{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-18T08:33:56.514855800Z",
     "start_time": "2024-07-18T08:33:56.438976400Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sys.version_info(major=3, minor=12, micro=3, releaselevel='final', serial=0)\n",
      "matplotlib 3.9.0\n",
      "numpy 1.26.4\n",
      "pandas 2.2.2\n",
      "sklearn 1.5.0\n",
      "torch 2.3.1+cpu\n",
      "cpu\n"
     ]
    }
   ],
   "source": [
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "from tqdm.auto import tqdm\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "print(sys.version_info)\n",
    "for module in mpl, np, pd, sklearn, torch:\n",
    "    print(module.__name__, module.__version__)\n",
    "    \n",
    "device = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "print(device)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 加载数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-18T08:33:56.676364900Z",
     "start_time": "2024-07-18T08:33:56.455548Z"
    }
   },
   "outputs": [],
   "source": [
    "from torchvision import datasets\n",
    "from torchvision.transforms import ToTensor\n",
    "\n",
    "# fashion_mnist图像分类数据集\n",
    "train_ds = datasets.FashionMNIST(\n",
    "    root=\"data\",\n",
    "    train=True,\n",
    "    download=True,\n",
    "    transform=ToTensor()\n",
    ")\n",
    "\n",
    "test_ds = datasets.FashionMNIST(\n",
    "    root=\"data\",\n",
    "    train=False,\n",
    "    download=True,\n",
    "    transform=ToTensor()\n",
    ")\n",
    "\n",
    "# torchvision 数据集里没有提供训练集和验证集的划分\n",
    "# 当然也可以用 torch.utils.data.Dataset 实现人为划分\n",
    "# 从数据集到dataloader\n",
    "train_loader = torch.utils.data.DataLoader(train_ds, batch_size=16, shuffle=True)\n",
    "val_loader = torch.utils.data.DataLoader(test_ds, batch_size=16, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-18T08:33:56.687439100Z",
     "start_time": "2024-07-18T08:33:56.525867500Z"
    }
   },
   "outputs": [],
   "source": [
    "from torchvision.transforms import Normalize\n",
    "\n",
    "# 遍历train_ds得到每张图片，计算每个通道的均值和方差\n",
    "def cal_mean_std(ds):\n",
    "    mean = 0.\n",
    "    std = 0.\n",
    "    for img, _ in ds:\n",
    "        mean += img.mean(dim=(1, 2))\n",
    "        std += img.std(dim=(1, 2))\n",
    "    mean /= len(ds)\n",
    "    std /= len(ds)\n",
    "    return mean, std\n",
    "\n",
    "\n",
    "# print(cal_mean_std(train_ds))\n",
    "# 0.2860， 0.3205\n",
    "transforms = nn.Sequential(\n",
    "    Normalize([0.2860], [0.3205])\n",
    ")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 定义模型\n",
    "\n",
    "这里我们没有用`nn.Linear`的默认初始化，而是采用了xavier均匀分布去初始化全连接层的权重\n",
    "\n",
    "xavier初始化出自论文 《Understanding the difficulty of training deep feedforward neural networks》，适用于使用`tanh`和`sigmoid`激活函数的方法。当然，我们这里的模型采用的是`relu`激活函数，采用He初始化（何凯明初始化）会更加合适。感兴趣的同学可以自己动手修改并比对效果。\n",
    "\n",
    "|神经网络层数|初始化方式|early stop at epoch| val_loss | vla_acc|\n",
    "|-|-|-|-|-|\n",
    "|20|默认|\n",
    "|20|xaviier_uniform|\n",
    "|20|he_uniform|\n",
    "|...|\n",
    "\n",
    "He初始化出自论文 《Delving deep into rectifiers: Surpassing human-level performance on ImageNet classification》"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-18T08:33:56.689162500Z",
     "start_time": "2024-07-18T08:33:56.534862300Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "               layer_name               \tparamerters num\n",
      "0linear_relu_stack.0.weight              \t  78400   \n",
      "1linear_relu_stack.0.bias                \t   100    \n",
      "2linear_relu_stack.1.weight              \t   100    \n",
      "3linear_relu_stack.1.bias                \t   100    \n",
      "4linear_relu_stack.Linear_1.weight       \t  10000   \n",
      "5linear_relu_stack.Linear_1.bias         \t   100    \n",
      "6linear_relu_stack.batchnorm_1.weight    \t   100    \n",
      "7linear_relu_stack.batchnorm_1.bias      \t   100    \n",
      "8linear_relu_stack.Linear_2.weight       \t  10000   \n",
      "9linear_relu_stack.Linear_2.bias         \t   100    \n",
      "10linear_relu_stack.batchnorm_2.weight    \t   100    \n",
      "11linear_relu_stack.batchnorm_2.bias      \t   100    \n",
      "12linear_relu_stack.Linear_3.weight       \t  10000   \n",
      "13linear_relu_stack.Linear_3.bias         \t   100    \n",
      "14linear_relu_stack.batchnorm_3.weight    \t   100    \n",
      "15linear_relu_stack.batchnorm_3.bias      \t   100    \n",
      "16linear_relu_stack.Linear_4.weight       \t  10000   \n",
      "17linear_relu_stack.Linear_4.bias         \t   100    \n",
      "18linear_relu_stack.batchnorm_4.weight    \t   100    \n",
      "19linear_relu_stack.batchnorm_4.bias      \t   100    \n",
      "20linear_relu_stack.Linear_5.weight       \t  10000   \n",
      "21linear_relu_stack.Linear_5.bias         \t   100    \n",
      "22linear_relu_stack.batchnorm_5.weight    \t   100    \n",
      "23linear_relu_stack.batchnorm_5.bias      \t   100    \n",
      "24linear_relu_stack.Linear_6.weight       \t  10000   \n",
      "25linear_relu_stack.Linear_6.bias         \t   100    \n",
      "26linear_relu_stack.batchnorm_6.weight    \t   100    \n",
      "27linear_relu_stack.batchnorm_6.bias      \t   100    \n",
      "28linear_relu_stack.Linear_7.weight       \t  10000   \n",
      "29linear_relu_stack.Linear_7.bias         \t   100    \n",
      "30linear_relu_stack.batchnorm_7.weight    \t   100    \n",
      "31linear_relu_stack.batchnorm_7.bias      \t   100    \n",
      "32linear_relu_stack.Linear_8.weight       \t  10000   \n",
      "33linear_relu_stack.Linear_8.bias         \t   100    \n",
      "34linear_relu_stack.batchnorm_8.weight    \t   100    \n",
      "35linear_relu_stack.batchnorm_8.bias      \t   100    \n",
      "36linear_relu_stack.Linear_9.weight       \t  10000   \n",
      "37linear_relu_stack.Linear_9.bias         \t   100    \n",
      "38linear_relu_stack.batchnorm_9.weight    \t   100    \n",
      "39linear_relu_stack.batchnorm_9.bias      \t   100    \n",
      "40linear_relu_stack.Linear_10.weight      \t  10000   \n",
      "41linear_relu_stack.Linear_10.bias        \t   100    \n",
      "42linear_relu_stack.batchnorm_10.weight   \t   100    \n",
      "43linear_relu_stack.batchnorm_10.bias     \t   100    \n",
      "44linear_relu_stack.Linear_11.weight      \t  10000   \n",
      "45linear_relu_stack.Linear_11.bias        \t   100    \n",
      "46linear_relu_stack.batchnorm_11.weight   \t   100    \n",
      "47linear_relu_stack.batchnorm_11.bias     \t   100    \n",
      "48linear_relu_stack.Linear_12.weight      \t  10000   \n",
      "49linear_relu_stack.Linear_12.bias        \t   100    \n",
      "50linear_relu_stack.batchnorm_12.weight   \t   100    \n",
      "51linear_relu_stack.batchnorm_12.bias     \t   100    \n",
      "52linear_relu_stack.Linear_13.weight      \t  10000   \n",
      "53linear_relu_stack.Linear_13.bias        \t   100    \n",
      "54linear_relu_stack.batchnorm_13.weight   \t   100    \n",
      "55linear_relu_stack.batchnorm_13.bias     \t   100    \n",
      "56linear_relu_stack.Linear_14.weight      \t  10000   \n",
      "57linear_relu_stack.Linear_14.bias        \t   100    \n",
      "58linear_relu_stack.batchnorm_14.weight   \t   100    \n",
      "59linear_relu_stack.batchnorm_14.bias     \t   100    \n",
      "60linear_relu_stack.Linear_15.weight      \t  10000   \n",
      "61linear_relu_stack.Linear_15.bias        \t   100    \n",
      "62linear_relu_stack.batchnorm_15.weight   \t   100    \n",
      "63linear_relu_stack.batchnorm_15.bias     \t   100    \n",
      "64linear_relu_stack.Linear_16.weight      \t  10000   \n",
      "65linear_relu_stack.Linear_16.bias        \t   100    \n",
      "66linear_relu_stack.batchnorm_16.weight   \t   100    \n",
      "67linear_relu_stack.batchnorm_16.bias     \t   100    \n",
      "68linear_relu_stack.Linear_17.weight      \t  10000   \n",
      "69linear_relu_stack.Linear_17.bias        \t   100    \n",
      "70linear_relu_stack.batchnorm_17.weight   \t   100    \n",
      "71linear_relu_stack.batchnorm_17.bias     \t   100    \n",
      "72linear_relu_stack.Linear_18.weight      \t  10000   \n",
      "73linear_relu_stack.Linear_18.bias        \t   100    \n",
      "74linear_relu_stack.batchnorm_18.weight   \t   100    \n",
      "75linear_relu_stack.batchnorm_18.bias     \t   100    \n",
      "76linear_relu_stack.Linear_19.weight      \t  10000   \n",
      "77linear_relu_stack.Linear_19.bias        \t   100    \n",
      "78linear_relu_stack.batchnorm_19.weight   \t   100    \n",
      "79linear_relu_stack.batchnorm_19.bias     \t   100    \n",
      "80linear_relu_stack.Output Layer.weight   \t   1000   \n",
      "81linear_relu_stack.Output Layer.bias     \t    10    \n"
     ]
    },
    {
     "data": {
      "text/plain": "275410"
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class NeuralNetwork(nn.Module):\n",
    "    def __init__(self, layers_num=2):\n",
    "        super().__init__()\n",
    "        self.transforms = transforms\n",
    "        self.flatten = nn.Flatten()\n",
    "        # 多加几层\n",
    "        self.linear_relu_stack = nn.Sequential(\n",
    "            nn.Linear(28 * 28, 100),  # in_features=784, out_features=300\n",
    "            # 一般而言，先batch norm 后 激活函数\n",
    "            # 参见论文 《Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift》\n",
    "            nn.BatchNorm1d(100),  # num of features=100\n",
    "            nn.ReLU(),\n",
    "        )\n",
    "        # 加19层\n",
    "        for i in range(1, layers_num):\n",
    "            self.linear_relu_stack.add_module(f\"Linear_{i}\", nn.Linear(100, 100))\n",
    "            self.linear_relu_stack.add_module(f\"batchnorm_{i}\", nn.BatchNorm1d(100))\n",
    "            self.linear_relu_stack.add_module(f\"relu\", nn.ReLU())\n",
    "        # 输出层\n",
    "        self.linear_relu_stack.add_module(\"Output Layer\", nn.Linear(100, 10))\n",
    "        \n",
    "        self.init_weights()\n",
    "        \n",
    "    def init_weights(self):\n",
    "        \"\"\"使用 xavier 均匀分布来初始化全连接层的权重 W\"\"\"\n",
    "        for m in self.modules():\n",
    "            if isinstance(m, nn.Linear):\n",
    "                nn.init.xavier_uniform_(m.weight)\n",
    "                nn.init.zeros_(m.bias)\n",
    "\n",
    "    def forward(self, x):\n",
    "        # x.shape [batch size, 1, 28, 28]\n",
    "        x = self.transforms(x)\n",
    "        x = self.flatten(x)  \n",
    "        # 展平后 x.shape [batch size, 28 * 28]\n",
    "        logits = self.linear_relu_stack(x)\n",
    "        # logits.shape [batch size, 10]\n",
    "        return logits\n",
    "\n",
    "print(f\"{'layer_name':^40}\\tparamerters num\")\n",
    "total_params = 0\n",
    "for idx, (key, value) in enumerate(NeuralNetwork(20).named_parameters()):\n",
    "    print(\"{}{:<40}\\t{:^10}\".format(idx,key, np.prod(value.shape)))\n",
    "    total_params += np.prod(value.shape)\n",
    "total_params"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-18T08:33:56.703151700Z",
     "start_time": "2024-07-18T08:33:56.581910200Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "@torch.no_grad()\n",
    "def evaluating(model, dataloader, loss_fct):\n",
    "    loss_list = []\n",
    "    pred_list = []\n",
    "    label_list = []\n",
    "    for datas, labels in dataloader:\n",
    "        datas = datas.to(device)\n",
    "        labels = labels.to(device)\n",
    "        # 前向计算\n",
    "        logits = model(datas)\n",
    "        loss = loss_fct(logits, labels)         # 验证集损失\n",
    "        loss_list.append(loss.item())\n",
    "        \n",
    "        preds = logits.argmax(axis=-1)    # 验证集预测\n",
    "        pred_list.extend(preds.cpu().numpy().tolist())\n",
    "        label_list.extend(labels.cpu().numpy().tolist())\n",
    "        \n",
    "    acc = accuracy_score(label_list, pred_list)\n",
    "    return np.mean(loss_list), acc\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-18T08:33:56.966378400Z",
     "start_time": "2024-07-18T08:33:56.699153500Z"
    }
   },
   "outputs": [],
   "source": [
    "from torch.utils.tensorboard import SummaryWriter\n",
    "\n",
    "\n",
    "class TensorBoardCallback:\n",
    "    def __init__(self, log_dir, flush_secs=10):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            log_dir (str): dir to write log.\n",
    "            flush_secs (int, optional): write to dsk each flush_secs seconds. Defaults to 10.\n",
    "        \"\"\"\n",
    "        self.writer = SummaryWriter(log_dir=log_dir, flush_secs=flush_secs)\n",
    "\n",
    "    def draw_model(self, model, input_shape):\n",
    "        self.writer.add_graph(model, input_to_model=torch.randn(input_shape))\n",
    "        \n",
    "    def add_loss_scalars(self, step, loss, val_loss):\n",
    "        self.writer.add_scalars(\n",
    "            main_tag=\"training/loss\", \n",
    "            tag_scalar_dict={\"loss\": loss, \"val_loss\": val_loss},\n",
    "            global_step=step,\n",
    "            )\n",
    "        \n",
    "    def add_acc_scalars(self, step, acc, val_acc):\n",
    "        self.writer.add_scalars(\n",
    "            main_tag=\"training/accuracy\",\n",
    "            tag_scalar_dict={\"accuracy\": acc, \"val_accuracy\": val_acc},\n",
    "            global_step=step,\n",
    "        )\n",
    "        \n",
    "    def add_lr_scalars(self, step, learning_rate):\n",
    "        self.writer.add_scalars(\n",
    "            main_tag=\"training/learning_rate\",\n",
    "            tag_scalar_dict={\"learning_rate\": learning_rate},\n",
    "            global_step=step,\n",
    "            \n",
    "        )\n",
    "    \n",
    "    def __call__(self, step, **kwargs):\n",
    "        # add loss\n",
    "        loss = kwargs.pop(\"loss\", None)\n",
    "        val_loss = kwargs.pop(\"val_loss\", None)\n",
    "        if loss is not None and val_loss is not None:\n",
    "            self.add_loss_scalars(step, loss, val_loss)\n",
    "        # add acc\n",
    "        acc = kwargs.pop(\"acc\", None)\n",
    "        val_acc = kwargs.pop(\"val_acc\", None)\n",
    "        if acc is not None and val_acc is not None:\n",
    "            self.add_acc_scalars(step, acc, val_acc)\n",
    "        # add lr\n",
    "        learning_rate = kwargs.pop(\"lr\", None)\n",
    "        if learning_rate is not None:\n",
    "            self.add_lr_scalars(step, learning_rate)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-18T08:33:56.973024800Z",
     "start_time": "2024-07-18T08:33:56.968377200Z"
    }
   },
   "outputs": [],
   "source": [
    "class SaveCheckpointsCallback:\n",
    "    def __init__(self, save_dir, save_step=5000, save_best_only=True):\n",
    "        \"\"\"\n",
    "        Save checkpoints each save_epoch epoch. \n",
    "        We save checkpoint by epoch in this implementation.\n",
    "        Usually, training scripts with pytorch evaluating model and save checkpoint by step.\n",
    "\n",
    "        Args:\n",
    "            save_dir (str): dir to save checkpoint\n",
    "            save_epoch (int, optional): the frequency to save checkpoint. Defaults to 1.\n",
    "            save_best_only (bool, optional): If True, only save the best model or save each model at every epoch.\n",
    "        \"\"\"\n",
    "        self.save_dir = save_dir\n",
    "        self.save_step = save_step\n",
    "        self.save_best_only = save_best_only\n",
    "        self.best_metrics = -1\n",
    "        \n",
    "        # mkdir\n",
    "        if not os.path.exists(self.save_dir):\n",
    "            os.makedirs(self.save_dir) #makedirs 创建多层目录\n",
    "        \n",
    "    def __call__(self, step, state_dict, metric=None):\n",
    "        if step % self.save_step > 0:\n",
    "            return\n",
    "        \n",
    "        if self.save_best_only:\n",
    "            assert metric is not None\n",
    "            if metric >= self.best_metrics:\n",
    "                # save checkpoints\n",
    "                torch.save(state_dict, os.path.join(self.save_dir, \"best.ckpt\"))\n",
    "                # update best metrics\n",
    "                self.best_metrics = metric\n",
    "        else:\n",
    "            torch.save(state_dict, os.path.join(self.save_dir, f\"{step}.ckpt\"))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-18T08:33:56.985531800Z",
     "start_time": "2024-07-18T08:33:56.974024500Z"
    }
   },
   "outputs": [],
   "source": [
    "class EarlyStopCallback:\n",
    "    def __init__(self, patience=5, min_delta=0.01):\n",
    "        \"\"\"\n",
    "\n",
    "        Args:\n",
    "            patience (int, optional): Number of epochs with no improvement after which training will be stopped.. Defaults to 5.\n",
    "            min_delta (float, optional): Minimum change in the monitored quantity to qualify as an improvement, i.e. an absolute \n",
    "                change of less than min_delta, will count as no improvement. Defaults to 0.01.\n",
    "        \"\"\"\n",
    "        self.patience = patience\n",
    "        self.min_delta = min_delta\n",
    "        self.best_metric = -1\n",
    "        self.counter = 0\n",
    "        \n",
    "    def __call__(self, metric):\n",
    "        if metric >= self.best_metric + self.min_delta:\n",
    "            # update best metric\n",
    "            self.best_metric = metric\n",
    "            # reset counter \n",
    "            self.counter = 0\n",
    "        else: \n",
    "            self.counter += 1\n",
    "            \n",
    "    @property\n",
    "    def early_stop(self):\n",
    "        return self.counter >= self.patience\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-18T08:35:07.904866200Z",
     "start_time": "2024-07-18T08:35:07.895930700Z"
    }
   },
   "outputs": [],
   "source": [
    "# 训练\n",
    "def training(\n",
    "    model, \n",
    "    train_loader, \n",
    "    val_loader, \n",
    "    epoch, \n",
    "    loss_fct, \n",
    "    optimizer, \n",
    "    tensorboard_callback=None,\n",
    "    save_ckpt_callback=None,\n",
    "    early_stop_callback=None,\n",
    "    eval_step=500,\n",
    "    ):\n",
    "    record_dict = {\n",
    "        \"train\": [],\n",
    "        \"val\": []\n",
    "    }\n",
    "    \n",
    "    global_step = 0\n",
    "    model.train()\n",
    "    with tqdm(total=epoch * len(train_loader)) as pbar:\n",
    "        for epoch_id in range(epoch):\n",
    "            # training\n",
    "            for datas, labels in train_loader:\n",
    "                datas = datas.to(device)\n",
    "                labels = labels.to(device)\n",
    "                # 梯度清空\n",
    "                optimizer.zero_grad()\n",
    "                # 模型前向计算\n",
    "                logits = model(datas)\n",
    "                # 计算损失\n",
    "                loss = loss_fct(logits, labels)\n",
    "                # 梯度回传\n",
    "                loss.backward()\n",
    "                # 调整优化器，包括学习率的变动等\n",
    "                optimizer.step()\n",
    "                preds = logits.argmax(axis=-1)\n",
    "            \n",
    "                acc = accuracy_score(labels.cpu().numpy(), preds.cpu().numpy())    \n",
    "                loss = loss.cpu().item()\n",
    "                # record\n",
    "                \n",
    "                record_dict[\"train\"].append({\n",
    "                    \"loss\": loss, \"acc\": acc, \"step\": global_step\n",
    "                })\n",
    "                \n",
    "                # evaluating\n",
    "                if global_step % eval_step == 0:\n",
    "                    model.eval()\n",
    "                    val_loss, val_acc = evaluating(model, val_loader, loss_fct)\n",
    "                    record_dict[\"val\"].append({\n",
    "                        \"loss\": val_loss, \"acc\": val_acc, \"step\": global_step\n",
    "                    })\n",
    "                    model.train()\n",
    "                    \n",
    "                    # 1. 使用 tensorboard 可视化\n",
    "                    if tensorboard_callback is not None:\n",
    "                        tensorboard_callback(\n",
    "                            global_step, \n",
    "                            loss=loss, val_loss=val_loss,\n",
    "                            acc=acc, val_acc=val_acc,\n",
    "                            lr=optimizer.param_groups[0][\"lr\"],\n",
    "                            )\n",
    "                    \n",
    "                    # 2. 保存模型权重 save model checkpoint\n",
    "                    if save_ckpt_callback is not None:\n",
    "                        save_ckpt_callback(global_step, model.state_dict(), metric=val_acc)\n",
    "\n",
    "                    # 3. 早停 Early Stop\n",
    "                    if early_stop_callback is not None:\n",
    "                        early_stop_callback(val_acc)\n",
    "                        if early_stop_callback.early_stop:\n",
    "                            print(f\"Early stop at epoch {epoch_id} / global_step {global_step}\")\n",
    "                            return record_dict\n",
    "                    \n",
    "                # udate step\n",
    "                global_step += 1\n",
    "                pbar.update(1)\n",
    "                pbar.set_postfix({\"epoch\": epoch_id})\n",
    "        \n",
    "    return record_dict\n",
    "        \n",
    "\n",
    "epoch = 100\n",
    "\n",
    "# 别 20 层了 2333，太深了没法训练\n",
    "model = NeuralNetwork(layers_num=10)\n",
    "\n",
    "# 1. 定义损失函数 采用交叉熵损失\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "outputs": [],
   "source": [
    "loss_fct = nn.CrossEntropyLoss()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-07-18T08:35:27.337869300Z",
     "start_time": "2024-07-18T08:35:27.327864600Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "\n",
    "# 2. 定义优化器 采用SGD\n",
    "# Optimizers specified in the torch.optim package\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=0.001, momentum=0.9)\n",
    "\n",
    "# 1. tensorboard 可视化\n",
    "tensorboard_callback = TensorBoardCallback(\"runs/bn\")\n",
    "tensorboard_callback.draw_model(model, [1, 28, 28])\n",
    "# 2. save best\n",
    "save_ckpt_callback = SaveCheckpointsCallback(\"checkpoints/bn\", save_best_only=True)\n",
    "# 3. early stop\n",
    "early_stop_callback = EarlyStopCallback(patience=10, min_delta=0.001)\n",
    "\n",
    "model = model.to(device)\n",
    "record = training(\n",
    "    model,\n",
    "    train_loader,\n",
    "    val_loader,\n",
    "    epoch,\n",
    "    loss_fct,\n",
    "    optimizer,\n",
    "    tensorboard_callback=tensorboard_callback,\n",
    "    save_ckpt_callback=save_ckpt_callback,\n",
    "    early_stop_callback=early_stop_callback,\n",
    "    eval_step=len(train_loader)\n",
    "    )"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-11-19T05:27:06.121149400Z",
     "start_time": "2023-11-19T05:27:05.780013Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "<Figure size 864x360 with 2 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtIAAAE9CAYAAAAmkIVgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAABxGUlEQVR4nO3dd3hUZfbA8e87k0kmvUKAhF6lI6EpSOxYsSOgggXWsnbdddf9bXF33XWLq66KgqDCIoptdXdxWSkBFem9hiIllARIIT0zmff3x50J6Y1JZu5wPs8zz0xm7tx7ZhIuZ86c932V1hohhBBCCCFE01h8HYAQQgghhBBmJIm0EEIIIYQQzSCJtBBCCCGEEM0gibQQQgghhBDNIIm0EEIIIYQQzSCJtBBCCCGEEM0Q5KsDJyQk6C5dujT5eYWFhYSHh3s/IC+SGL1DYvQOidE7qse4YcOGU1rrNj4MqVXJOdu3JEbvkBi9w4wxttg5W2vtk8vQoUN1cyxfvrxZz2tNEqN3SIzeITF6R/UYgfXaR+dPX1zknO1bEqN3SIzeYcYYW+qcLa0dQgghhBBCNIMk0kIIIYQQQjSDJNJCCCGEEEI0g88GGwohWo7D4SAjI4OSkhKio6PZtWuXr0Oqlz/HaLfbSU5O9nUYQggh/JAk0kIEoIyMDCIjI+nSpQsFBQVERkb6OqR65efn+2WMWmtOnz5NRkaGr0MRQgjhh6S1Q4gAVFJSQnx8PEopX4diakop4uPjKSkp8XUoQggh/JAk0kIEKEmivUPeRyGEEHWRRFoIIQKMUmqOUipLKbW9jseVUuo1pdQ+pdRWpdSFrR2jEEIEAkmkhRBel5uby5tvvtnk51177bXk5uY2+XlTp07lk08+afLzAth7wLh6Hr8G6Om+TAdmtEJMQggRcEyVSKftyWJDptPXYQghGlBXIu101v/vd9GiRcTExLRQVOcPrfVKILueTcYDc90Lfq0GYpRS7VsnuvPPxsM5HM0t9tnxtdak7ckiv8Thsxi8YdPhHE7kyXgF4V9MNWvHe6sOcui4g6d9HYgQol7PPfcc+/fvZ/DgwdhsNux2O7GxsezevZv09HRuuukmjhw5QklJCY8//jgTJ04EoEuXLqxfv56CggKuueYaRo8ezapVq0hKSuKLL74gNDS0wWMvXbqUZ555BqfTybBhw5gxYwYhISE899xzfPnllwQFBXHVVVfxl7/8hY8//pjf/OY3WK1WoqOjWblyZUu/Nf4iCThS6ecM933HK2+klJqOUbEmMTGRtLS0Jh+ooKCgWc9rTS0Z46pjTmZtLSUyWPHccDsdIppXv2pujFprPtxdxuJDTrpGWXhmmJ1wW8v0/bfk+5hT4uLZlcV0j7bwsxENnwfqcr7/PXqLxHiWqRLpmFAbOxza12EIYSov/W8/e095txrWt0MUv7qhX52P//GPf2T79u1s3ryZtLQ0rrvuOrZv307Xrl0BmDNnDnFxcRQXFzNs2DCuuuqqGtPf7d27lwULFjBr1izuuOMOPv30U+6666564yopKWHq1KksXbqUXr16cc899zBjxgzuvvtuPv/8c3bv3o1SqqJ95IUXXmDx4sUkJSU1q6Uk0GmtZwIzAVJSUnRqamqT95GWlkZznteaWirGf246yjuLN5PSJZYfThXx8mYXH04fTo+2Ea0So9aa3/9nF4sP/cAVFySyIj2Lt/fYmHf/CKJDbU2OoSVibKxff7kDp+sge3JchHQcwKju8c3az/n89+hNEuNZpmrtiAkLplASaSFMZ/jw4RVJNMBrr73GoEGDGDlyJEeOHGH//v01ntO1a1cGDx4MwNChQzl48GCDx9mzZw9du3alV69eAEyZMoWVK1cSHR2N3W7n/vvv57PPPiMsLAyAiy++mKlTpzJr1izKy8vP/YWax1GgY6Wfk933CS/5YvNRnlq4mRFd45l73wg+nD4CgImzVrMvq6DFj6+15sVFu3jn2x+YelEXZt0zlLfuGsqu42e4Z/Ya8orN0+aReaaED9YeZvzgDrSJDOHVpem+DkmICqaqSEeF2ih2QrlLY7XIlFRCNMZPr+ru88VOwsPDK26npaWxZMkSvv/+e8LCwkhNTaW0tLTGc0JCQipuW61WioubX1UPCgpi7dq1LF26lE8++YTXX3+dZcuW8dZbb7FmzRr+85//MHToUDZs2EB8fPMqXSbzJfBjpdSHwAggT2t9vIHniEb6YvNRnvxoM8O7xjF7agqhwVZ6tI1kwbQRTJy1momzVvPh9JF0b9P0ynRjaK35w1e7mfXND0wZ1Zlf3dAXpRSXX5DIjMlDeWj+Bu6Zs5Z59w8nyu79yrS3zUjbT7lL8/SVvRmYnMlv/72T1QdOM7LbefFvVfg5c1WkQ21oMP2ACSECXWRkJPn5+bU+lpeXR2xsLGFhYezevZvVq1d77bi9e/fm4MGD7Nu3D4B58+YxduxYCgoKyMvL49prr+Vvf/sbW7ZsAWD//v2MGDGCF154gTZt2nDkyJH6dm8aSqkFwPdAb6VUhlLqfqXUg0qpB92bLAIOAPuAWcDDPgo14HiS6GFd4pgzdRhhwWfrVT0TI/lg2khcLs3Emas5cNL7lWmtNX/8ajczVx7g7pGd+fWN/arMhX5F30TemHQhO4/lcc/stZzx8/9PPdXoWy9MolN8GJNHdDKq0kv2+jo0IQCzJdJhxifn3CL//ocvxPkuPj6eiy++mP79+/Pss89WeWzcuHE4nU4uuOACnnvuOUaOHOm149rtdt59911uv/12BgwYgMVi4cEHHyQ/P5/rr7+egQMHMnr0aF5++WUAnn32WQYMGED//v256KKLGDRokNdi8SWt9UStdXuttU1rnay1nq21fktr/Zb7ca21fkRr3V1rPUBrvd7XMQeCf205xpMfbSalSxzv3ls1ifbolRjJgukjKXdpJs5azQ+nCr12fK01L/13D2+vPMBdIzvxwvh+tS4odFW/drwx6UK2H81jypy1fl2cemuFUY3+8aU9AbDbrPzokm58f+A0aw6c9nF0QpistcMzOCLXRL1dQpyvPvjgg1rvDwkJ4auvvqpyn6d67emDTkhIYPv2s2uJPPPMM/Ue67333qu4ffnll7Np06Yqj7dv3561a9fWeN5nn31W736FaKx/bz3GEx9tJqVzHO9OrT2J9ujlrkxPmrWaO2d+z4fTR9E1IbzO7RtDa82fFu/hrRX7mTyiEy/c2L/eVTmv6teONyZfyCPzN3LPnLXMvW84kX7W5pF1poQP1hzmliFGNdpj8ojOvLXiAK8u3csH0t4hfMyUFWkzDZIQQggR2P6z9TiPf7iZoZ1ieffeYYSHNFyj6t3OSKYd5Uabx8FzqExrrfnz4j3MSNvPpBGd+O34/lgaMY7o6n7teH3ShWzL8M/K9FsrDuB0aX58WY8q94cGW3lwbDdW7T/NuoP1TZcuRMszVSIdHRoMQG5RmY8jEUL4wiOPPMLgwYOrXN59911fhyXOY//ZepzHPtzEhZ1iGp1EexjJ9AhKneVMnLWaQ6ebnkxrrfnL//bwZtp+Jg7vxO8amUR7jOvfjtcnDWFLRh5T311HQal/LHqWlV/C/DWHuGlwEp3ja1brJ4/oTEJEsPRKC58zVSItFWkhzm9vvPEGmzdvrnK59957fR2WOE8t2mYk0UM6xvDuvcOblER79GkXxQfTRlLiKOfOmU1LprXW/PV/6byxfD8Th3fk9zc1LYn2GNe/Pa9PHMLmI7lMnbPWL5Lpt93V6EerVaM9QoOt/OiS7ny77xTrpSotfMhUibSnRzpPBhsKIYTwoa+2HefRBUYS/d59w4loRhLtcUH7KOY/YCTTE2eu5vDpogafo7Xmb1+n8/ryfdw5rCO/v2lAs5Joj2sGtOfvE4ew6Ugu977r22T6ZH4p89ccYvzgDnSpp3d88shOxIcH8+pSqUoL3zFVIm2zWrBbZbChEEII3/nvdiOJHuyFJNqjbwcjmS5yGG0eR7LrT6b/tmQvry0zkugXbz63JNrj2gHtee3OIWw8bCTThT5Kpmeu3E+Z08Wjl/Wsd7uw4CB+NLYb3+w9xYZDUpUWvtFgIq2Usiul1iqltiildiilflPLNiFKqY+UUvuUUmuUUl1aJFog3KZk+jshhBA+8d/tJ/jxB5sYmBzNe/cO80oS7WEk0yMoLHNy58y6k+m/fZ3Oa0v3ckdKsteSaI/rBrbn1TsHu5Ppda2eTJ/ML2XeaqM3ujEzmdw1sjPx4cG8Ir3SwkcaU5EuBS7TWg8CBgPjlFLVJ369H8jRWvcA/ga85NUoKwmzKemRFkII0eoW7zjBjz/YyIDkaN5voeni+nWI5h/3j6CgtPZk+pUl6by6dC+3D03mj7cM9GoS7XH9wA68MmEwGw7ncO976ygqa71ketY3ByhzumrM1FGXsOAgpl/iqUrntHB0QtTUYCLtnrjfs/ySzX3R1TYbD7zvvv0JcLmqbwLLcxBhg7ximbVDiEDSvn37Oh87ePAg/fv3b8VohKjpfztO8Mj8jfRParkk2qN/UjTzHzibTGfkGMn0q0v28sqSvdw2NJmXbm2ZJNrjhkEd+NuEwaw/mM2977ZOMn2qoJR53x9i/OAkujVh+fS7R3UmTnqlhY80qkdaKWVVSm0GsoCvtdZrqm2SBBwB0Fo7gTygRWZJl9YOIYQQrenrnZk88oGRRM+9fzhRrbBwiSeZzi9xcOfM1Xy4u5S/LUnn1gtbPon2uNGdTK87mM19rVCZnrXyAKXO8kZXoz08VemV6SfZeFiq0mbg70vTN0Wjmru01uXAYKVUDPC5Uqq/1np7A0+rQSk1HZgOkJiYSFpaWlN3QTBOsvIKm/Xc1lJQUODX8YHE6C3+GmN0dHTFaoHBS3+J89ROr+7f1bYfpZfWGC5R4Ve/+hVJSUlMnz4dgBdffJGgoCC++eYbcnNzcTgc/N///R/XXXddxXM88VZXUFCAy+UiPz+fkpISnnzySTZt2kRQUBAvvvgil1xyCbt27eKhhx7C4XDgcrmYN28e7du3Z8qUKRw7dozy8nJ+8pOfcOuttzbr9ZaUlPjt71q0rCU7M3l4/gb6dmi9JNrDSKZHMvmd1fw3x8ktFybxp9sGYm2FJNpj/OAkAJ78aDP3v7eeOVOHERps9fpxTheUMvf7Q9w4qAPdm1CN9rh7ZGdmrjzAq0v28v59w70eX6A4VVDKw/M3ktI5lmev7l3v6pctZfvRPO6Zs5YXxvfj+oEdWv343takURJa61yl1HJgHFA5kT4KdAQylFJBQDRwupbnzwRmAqSkpOjU1NQmB7xwz/8oPl3O2LFjffIH0BhpaWk057W1JonRO/w1xl27dhEZGQlAmUURZPXegCgAbMEEu/dfm7vvvpsnnniCp59+GoAvvviCxYsX8+yzzxIVFcWpU6cYOXIkEyZMqPh3HFnH/iIiIrBYLERGRjJz5kyCg4PZsWMHu3fv5qqrriI9PZ158+bx1FNPMXnyZMrKyigvL2fRokV06tSJxYsXA5CXl1fnMRpit9uJiIjwy9+1aDlLd2Xy0PwN9G0fxdz7WjeJ9hiQHM1HPxrF3MVr+N1tg1o1ifYYPzgJreGphZu5//11zJ7i/WR65jcHKHGW8+MGZuqoS3hIEA+M6cqf/ruHTYdzGNIp1qvxBYLTBaVMmrWafVkFrP0hm1Kni19cd0Gr5lI7juVx1+w1hAcHMSg5ptWO25Ia/N9VKdUGcLiT6FDgSmoOJvwSmAJ8D9wGLNNaV++j9opwG5Q5XZQ4XC3yqViIQFN66W/qTXpbwpAhQ8jKyuLYsWOcPHmS2NhY2rVrx5NPPsnKlSuxWCwcPXqUzMxM2rVr1+j9fvvttzz66KMA9OnTh86dO5Oens6oUaP4/e9/T0ZGBrfccgs9e/ZkwIABPP300/z0pz/l+uuvZ8yYMS31ckUAWrY7k4f+sdFIou8fUbGOgS9c0D6Kq7vYfJJEe9w0JAmN5umFW3hg7jreucd7yXR2YRnzvj/EDQM70KNt06vRHveM6sKslQd4dele3rtXqtKVGUn0Gg5nFzHv/hF8vTOT2d/+gEXBz69tnWR6x7E8Jr+zhjCblQXTRtIxLqzFj9kaGtMj3R5YrpTaCqzD6JH+t1LqBaXUje5tZgPxSql9wFPAcy0TrtEjDZArAw6F8Gu33347n3zyCR999BETJkxg/vz5nDx5kg0bNrB582YSExMpKSnxyrEmTZrEl19+SWhoKNdeey3Lli2jV69ebNy4kQEDBvCLX/yCF154wSvHEoFv+e4sHpy3kT7tI32eRPuTm4ck85fbB7Fq/2mmzV1PiaPcK/ud9c0Bih3lPHZ503qjq4sICeKBMd1I23OSzUdyvRJbIDhdUMrkd9Zw8HQhs6cM4+IeCfzqhr5MGdWZWd/8wB++2k0L1T4r7Dx2piKJ/nD6KDrFB0YSDY2oSGuttwJDarn/l5VulwC3eze02nkS6bxiB+2jQ1vjkEKIZpgwYQLTpk3j1KlTrFixgoULF9K2bVtsNhvLly/n0KFDTd7nmDFjmD9/Ppdddhnp6ekcPnyY3r17c+DAAbp168Zjjz3G4cOH2bp1K3369CEuLo677rqLmJgY3nnnnRZ4lSLQLN+dxY/mbaB3u0jm3SdJdHW3XJiM1vDMJ1t44P31vDMlBbut+ZXp7MIy3l91kOsHdqBH23P/5mzKRV2Y9c0BXlu6lzlTh53z/swuu7CMye+s4YdThcyZaiTRAEopfn1jPzQwc+UBFPDcNX1apDK96/gZJr+zmlCblQXTRwZUEg1N7JH2BxUVaZm5Qwi/1q9fP/Lz80lKSqJ9+/ZMnjyZG264gQEDBpCSkkKfPn2avM+HH36Yhx56iAEDBhAUFMR7771HSEgICxcuZN68edhsNtq1a8fPf/5z1q1bx7PPPovFYsFmszFjxowWeJUikKTtMZLoXu0i+Mf9I4gOkyS6NrcOTUYDz36yhWlz1zPrnuYn0+94qtFNnKmjLhEhQUwb040/L97DliO5DOoY45X9mlFOYRmTZq3mh1NnK9GVKaX4zY390BreXnkAFDw3zrvJ9O4TRiU6JMjKh9NH0jm+4UV2zMaEibRxLYm0EP5v27ZtFbcTEhL4/vvva93u+PHjde6jS5cubN9ujG222+28++67NbZ57rnneO65qh1lV199NVdffXVzwhbnoRXpJ5k+bwM9EyWJbozbhiajteYnn26tSKabKsddjb5uQHt6JnpvHMc9ozpXVKVnn6dV6ZzCMia9s4YDpwqZPSWF0T0Tat1OKcUL4/uh0by94gAKxU/HeWc2j90nzjBp1hqCrZaATaLBlIm08cs9I6sbCiGE8IJtJ538fcl6erSJYP4DI4gJC/Z1SKZwe0pHNPDTT7cyfd4G7urctD7bd749QJGjnMcub95MHXWJtNt4YHRX/vK/dLZm5DIwQGaHaKwcdzvH/pMFvHNPCmN6tql3e6UUL9zYH63hrRX7UQp+co5T4+05kc+kWWuwWRULpo+kSyOWezcr0ybSMthQiMCybds27r777ir3hYSEsGZN9fWfhPCelekneXVTKb0SoySJboY7UjqChp9+tpWcbCuXXFLeqDYPoxp9iGsHtKeXF6vRHkav9A+8tnQv70w5f6rSuUVl3DV7DftOFjDrnhQu6VV/Eu1hsSh+O74/GpiRth+Lgmeual4ynZ6Zz6RZq7FZFR9OH0XXAE6iwYSJtN0KQRZZ3VCIQDNgwAA2b97s6zDEeeSbvSeZNnc97cMtzH9gBLHhkkQ3xx3DOgLwk0+38uA/NvDWXUMbTKZnf/sDBaVOHmvmvNEN8VSl//p1Otsy8hiQHN0ix/EnuUVGJXpvVgEz7x7K2EYm0R4Wi+J34/ujteaN5ftRKJ6+qleTkum97iTaalEsmDYy4JNoaOQS4f5EKUVMmI1cae0Qol4tPZ3R+ULex8D07d5TPPD+eromhPOTYXZJos/RHcM6cm+/YNL2nOShf2yg1Fn31Hi5RWW8t+og1w5oR+92LTfH/ZSLuxBlD+LVpXtb7Bj+Iq/IwV2z17A300iiU3u3bdZ+LBbF728awJ3DOvL68n28/HV6o8+BezPzmThrNRal+HD6SLo1Y4VKMzJdIg0QFWojTxJpIepkt9s5ffq0JIHnSGvN6dOnsdvtvg5FeNF3+05x//vr6JoQzgfTRhIZ7J+r5JrN2I42/nDLAJbvOclD/9hYZzJdUY32cm90dVF2Gw+M6caSXZlsP5rXosfyJU8SnX6igLfPIYn2sFgUL95sJNN/X7aPvzUimd6Xlc/EWWtQyuiJbpEk2uUCZyk4ir2/73NgutYOgJhQG3nS2iFEnZKTk8nIyODkyZOUlJT4fSLozzHa7XaSk5ObNe+18D+rqiXRcVKJ9qqJwzuhNfz88208/I+NvHnXhYRYLeBuD8grcvDedwe5pn87+rSLqrmDomw4tgmikyG+J1jOrd439eIuvPONsdphc2YWwVkGB1fCwW+h/SDoeRUEt0C7Qs5ByNoFsV1RrjryG2cZHNsIh76Dg9/Bia24NDhKXLxZbiUuLpzwZaGwIhiiOkBsV4hzX2K7QkwnsDZuNhrL8Y38wTqb+xLSWbqyHf85fTHXj7vW2Ee1Vo99WQXcOXMNSsGCaSPpfi5JtMsFu/8F37wMuYfB5YTyMih3gHZ/MOtxJdz1SfOP4WXmTKTDgsnK986KaEIEIpvNRteuXQFIS0tjyJAaayr5FTPEKMxv1b5T3Pf+OjrHhTP/gRGSRLeQSSM6odG88PlGvnntDS4vXISK7QztBrA1vz2DHRE8edEE0NpIlg6vhsPfG5eTu8/uyB4DHYe7LyMgaShYbJB/DPKOQl4GnMkwrm1hkHQhdLgQYrtUJHtRdhv3j+7G35akN74qXVYE+5fCrn/Bnv9CaR6gAA1BodDjcuh7E/S6Guy1fBhoihPb4NtXYMdnoF0AXIIFtnWC+B7Gh4mQCOM9ylgPTnc1ts0FlHW/iqXpOeQ7ixnTNYrwyCAj6XSUQPYPsH/52e0BlBU6DIEeV0DPK43blkq97I4S2PE5rJsFRzeggiPoGdeVboVfEbTnX7DnOQiNhQ5D6FYWDeH7OEZbfrY4mxDdhvenj23+Eu9aQ/piWP57OLEVEnpBv5vBGmwk/1abcdsSBPHdm3eMFmLKRDo61MberHxfhyGEEMIkvt9/mvveX0enuDDmTxtBfESIr0NqmKscHEUQ7Ce9plrDmWNG8hXZrt5NJ3ct5tqE3xN7Zi+rwy9jeJwdDq9lzJkjjAkG5v4RgiOhzP1/eUiUkSwPuB2SU4xE+chqOLIW9v7P2EZZjBio1mYQGmskv+Wl7p/jjKQ7aSgkD2NqyoW8820Qry3dy6ROdQScnwn7lkD6f41rR5Gx3wtuMC5dx8DRjbDrS9j5Jez+t5HYdUuF8DZQVmg8p6wIHIVG+0FEovEhIHm48ZrC4s6+j4e+MxLofV8bv9+RD0Of6yHvCIc2LKFLhANO74ND3xv7bdcfhk6FLhdDp1GcsUZz9ztr2Jl/hrfuGkr7CxJr/33ln4CcH4zE+vReo7K+4iVY8Ufjfep+mfHB4NRe2Pg+FJ02kvdr/gyD7kTZo7CUlfDqh1+QuWcNkxJP079wP8mZK+HI53QAPvYc770E45uEiESIaGtcwttCRBvjOjwBwhKM98GTwGsNB5bDst/D0fXGh6Cb3zb+DizNXzGzNZk2kZZZO4QQQjTG6gOnue89I4n+YNpIEupKol3l5/6f95ljRiJWXgYh0WCPNqqWIVHG7aAQQJ39elwpI0EsPGVUY7N2ua93w6l0IzkMjmC4NRoOdYeoJIhsD2Hx4CypmryVFRlVzX43Q5/r6n8tjmLYOBe+f934uW1faHvB2ev4nkY189gmOLoBjrqvC04Y8fa6BkZMh65jq37VrzWsnwOLf05scARLh77B/d/FckXbRHr1jeAfaVv48rZoujj2w+n9xrE6jTSOWz3eIZON6+Icoxqbsc5476KTIToJojsaLQzB4cZX/5k7jNaHoxuMpHf/UtAuopWFJeG9+TK9Ky4GQtFA43eRsd5I0vd9Dce3GMeKbA+DJxnJc+fRYK2UJnUdY1zGvWTEsutL2PMVZO6E4DCjKh4cbiSLNrvRrvHNXysqzST0MpLqU3uM54clwGW/gGEPGEk7AKM4mN2WLqmpZ99PZwnYQs/+iZU4uHv2WnYeP8OMyUO5vLYk2vO3FdXeuHS+6Oz9RdmwfxnsW2r8rW7/xPid9r4Whk+r8Tu1BNt59K47+Mmnfbh+QwZPXdmLNkmHmLclm3Y6kxdTo2mnsyDvCOQeMb4xOL4FCk+ebceoGhiExhiv32I1/t6jkuGGV2Hw5Ea3n/gLUybSMWE28kuclLs0VosMEhFCiED15ZZjZOY1v5Wv1FnOG8v3kxwbWjOJdrng+Ca6/DAfdj0PmduN6mBoHITFGslNaJxRSYvvCW37QJsLjJ89iYbWkLUTdi+CPf8xEs9zFd0R2vSB7qlGslGQScH+LYQ5SoyKYv5xo3cUjK/rg8PPJnFlBUabQGwXo8o5eLLRGuBRVgQb3oXvXoWCTOh0kZGMZu0yEitPf66ynE0AwXj93VKN9omCTNjwnvF6E3obydegiQQ5zsBHdxnV2u6XwU1vcXlkIr9NOMj/fbGDJbsyubpfd7qkNLFXOTTWaEXoeWXd21ht0GGwcUm5z7ivtMCoch5aRfyBb7in8GtCDiyCP71k/J7L8o33r+NwuPyXRu9tuwE1eoBrsFig0wjjcvXv69+2tMBI7o+shYx1OHf9h0JLBNt6/Yz09jdSjh3W5QA5FU/Z94ODvZYDde7y39uOs/NYHm9OHsoVfetIousTFgcDbjMuLpfx9xsaY3xAqYPFonjp1oFoDS9/nY7dCuH2WF6dfg3t6poH3OUyPgQVZkFBFhSdgsLT7utTxnXJGeP3NXSq+0Om+ZgykY4ONT6tnCl2yJRFQggRoE7ml/LYgvoTUyvllGPB6GGtXZ92kcy7f4SRRDvLjGrcnkXG1/gFmXTGYlTsxjxjVHiLc4yqXXG2UWErPAmlZ87uMDTOqKRGdzT6enPdA1GTUoyErPe1xjalZ6Akz7h4bjvLAH22RcFzbY82kvQ2vSCkZmKyMy2Ntp4qpctl7M8WarQXVE78XOVGIvv9G/DVT4yvzFOmwpC7jde86u/G6+kyBm6bA11Gn31uucOoEmftNBJrm91oj2g/2Ei0KrvkJ0Y/7dq3YdEzsPQFhrssUF4IV78IIx6qGCh496guaODVJXt54ope9f4+vSokwkj+u6USdCnM/Ho7y5ct5smeWVzUtsx4D7pfWqka3EIxdL0Eul7Ct3tPcf/OtZQ6tZE3b/2h7uft2VXnQ3abhTcmXciVzUmiq7NYjLaRRrBaFH+6bSBBFsX/tmWwYPrI+pd2t1ggPN64tL3g3GP1U6ZMpGPCjEQ6VxJpIYQIWKcLjZ7Xl+8YxFX9qvXkahdB2xYQvPwFXO0GUXLL+1W+/q4szGbFYlFGdXD+7XB4ldGf2/MK6H0t32WGMvrKG+oOxNNrenKX0XLhud6/zKiAjn4Sel9Ts2840guJTm0slpqJbcVjVug73rgcWWe0bqz6u1GBBuh2KYz9SdWv+j2sNqPq3rZPwzHY7DB4Igy602ilWPM2hUd2ETzhDWN2i2ruGdWFu0d2Pqdlp8/Vg5f3Y9Xuo0xKd/Jkp1483r9lp9+r7Nu9ntliIpgzdRhRoXW3L3zzzTeMGTOmzsdtVkVIkG/6h60WxUu3DeTq+NMtsiKlGZkzkQ41kufcojIg8FfNEUKI81F2YRkA7aNDiQip9N/Vie3wn6eNwWiJ/bEcWEbEZ3fDxAV1JtMVSfSRNXDj6zBwAgQZ/5c409LqD6Ryr2n3y7zwylpJx2HQ8X3IOWRUjztfZLQxeJNSxkC65BS2pKWRWksSfXZT37ZiWiyK+/oHk5jYjr8tSUcpWnwua6g6b/n8Bxoe6BoapKr+vfshi49/l/7Ev39TdYiuVJEWQggRmHIKjXN8bLi7eleaD2l/hNUzjIrs+Ddh0ETY+iH882FYcCfcucAY+FVZWSF8MMFIvG99B/rf2rovxNdiO8PoJ3wdhV+wKKM9QaN5+et0FPBoCybTnnnLu8Q3LokW5mPORLpSj7QQQogAorUxYK40n/KsPQxW+2iXZYWMY7DiT0aLxdCpRi+yZzqxwZMABf98CBZMgIkfnU2mPUn04VVwy6zzL4kWNVgtij/fNgg0/PXrdCwWxSOX9vD6cVbtP2W+KRdFk5kykY5xJ9IyBZ4QQgQIrWHdO7DkNxVzC98I3BgCfObept1AmPAPo5WgusETjTaDzx+ED+6ASR8BykiiD30HN880ZikQAncyffsgNPDnxXsAvJpMf7+/kVMuCtMzZSIdLYm0EEIEjjPH4YtHjHl/u19mDIgLieSTHfksOVDEW/enGrNaJPSqf27kQXdiVKYfNBJoZXEn0W/DwNtb69UIk7BaFH+5fRBaa/68eA9KwcOp555Me+Yt7xgrSfT5wJSJdJDVQkRIEHnS2iGEEOa2/TP495PGAibXvWzMKeseyPTd/s3sCM82FuxorEHuBPrz6UaV++a3YOAdLRS8MDurRfHXOwajgT/9dw8WpXhwbPOXoF594DT3vruOpNrmLRcByZSJNLhXNywu83UYQgghmqM4FxY9C9sWGnMV3zwTEqpWA7MLy4gNa8YUpwNvN+audZXXv4iHELiT6dsHoTX88avdKOBHzUim17iT6A4xdj6YNoI2kZJEnw9Mm0jHhNnIk9YOIYQwnzPH4Z3LjYGDlz4Po5+quhSzW25RMxNpMNc0dcLngqwWXr7D6Jn+w1e7UQqmX9L4ZHrtD9nc+56RRC+YPpK2kfaWC1b4FdMm0tGhNmntEEIIMzr0HZw5CpM/NRZFqUN2URnd2kTU+bgQ3hRktfC3O4ye6RcX7UahmHZJtwaft+5gNlPfXUu7aDsLpkkSfb4xbSIdE2YjPbPA12EIIYRoKmeJcd2m/uWicwsdFSvZCtEagqwWXplg9Ez/ftEulIIHxtSdTK8/mM3UOUYS/eG0kbSNkiT6fGPaRDo6NFhm7RBCCDPyJNJBdScdZU4X+aVO4prb2iFEMwVZLbw6YTBo+N1/dgG1J9PrD2YzZc5aEqMkiT6fmTaRjgmzkVdchtba58uOCiGEaAJnqXFdTyKdW2QMJo8Jl0RatL4gq4VX7hyMRvO7/+xCKcX9o7tWPL7hkJFEt41y90RLEn3eMm0iHR1qw1GuKXaUExZs2pchhBDnH0excV1PIp3j/sZRKtLCV2xWC6/eOQStN/Hbf+/EouDei7uy4VAOU+asM5LoaSNJlCT6vGbaDLTy6oaSSAshhIk4SwEF1rr7n7MLjYp0bLj0SAvfsVktvDZxCI9+sInf/GsnR7KLWbj+CAkRwSyYNpJ20ZJEn+8svg6guTwDUKRPWgghqlJKjVNK7VFK7VNKPVfL452UUsuVUpuUUluVUte2aoDOEqMaXU9bnqe1o9nT3wnhJTarhb9PGsLV/RKZ890PxEcEs2C6JNHCYNpSbpS7Ii1T4AkhxFlKKSvwBnAlkAGsU0p9qbXeWWmzXwALtdYzlFJ9gUVAl1YL0lkKtvqTkGx3Ih0nPdLCD9isFv4+8UI+Wn+Eq/omSjuHqGCuRNpRgtVZBEBMqHFyzZPVDYUQorLhwD6t9QEApdSHwHigciKtgSj37WjgWKtG6Cyutz8aIMfd2iHT3wl/ERxk4e6RnX0dhvAzDbZ2KKU6ur8C3KmU2qGUeryWbVKVUnlKqc3uyy9bJNoFdzJw668Bae0QQog6JAFHKv2c4b6vsl8DdymlMjCq0Y+2TmhuzlIIqn/55JwiB+HBVkKCrK0UlBBCNF1jKtJO4Gmt9UalVCSwQSn1dbWvCQG+0Vpf7/0QK7GFYS03qhTR0tohhBDNNRF4T2v9V6XUKGCeUqq/1tpVeSOl1HRgOkBiYiJpaWlNPlBBQUGN5/U9foTw0nLW1bO/XQdKCbW6mnXMpqotRn8jMXqHxOgdEuNZDSbSWuvjwHH37Xyl1C6M6kb1RLrl2UKxuIz5R8OCrdisilxJpIUQorKjQMdKPye776vsfmAcgNb6e6WUHUgAsipvpLWeCcwESElJ0ampqU0OJi0tjRrPOzYDguJr3l/J+z+spb0qIzV1dJOP2VS1xuhnJEbvkBi9Q2I8q0mzdiilugBDgDW1PDxKKbVFKfWVUqqfN4KroVIirZSS1Q2FEKKmdUBPpVRXpVQwcCfwZbVtDgOXAyilLgDswMlWi9DRcI90dpGDWBloKITwc40ebKiUigA+BZ7QWp+p9vBGoLPWusA9jdI/gZ617OOcvibskZVNW2dpxfOCdRl7Dx0lLe10k/bT0uQrD++QGL1DYvQOM8QIoLV2KqV+DCwGrMAcrfUOpdQLwHqt9ZfA08AspdSTGAMPp2qtdasF2Zge6cIyusSHtVJAQgjRPI1KpJVSNowker7W+rPqj1dOrLXWi5RSbyqlErTWp6ptd25fEzqWUX7864pSfftdqwixWUhNHdm0/bQw+crDOyRG75AYvcMMMXporRdhDCKsfN8vK93eCVzc2nFVcJaAPbreTXKKymQOaSGE32vMrB0KmA3s0lq/XMc27dzboZQa7t6v98vEtjCsrjJwGeNhYkJt0tohhBBm00BF2lHuIr/EKXNICyH8XmMq0hcDdwPblFKb3ff9HOgEoLV+C7gNeEgp5QSKgTtb5GtCT0+dswSCw4gOs7H7RL7XDyOEEKIFOUvAFlrnw54CSazMIS2E8HONmbXjW6DudVyNbV4HXvdWUHWyufvlHMVGIh1q44zM2iGEEObiLKm3Ip3jWR5cKtJCCD/XpFk7fM5TwXCcXd0wv9SJo9xVz5OEEEL4FWdJvbN2ZLtXNZQeaSGEvzNnIu0sAc6ubihVaSGEMJEGeqRziySRFkKYgzkTaXdFWlY3FEIIk9HaXZGuu0c6u9A4p8tgQyGEvzNpIl0MQLS7Ii2rGwohhEmUO0C7GtUjHSODDYUQfs5kibRnsKGnR9pdkZYp8IQQwhzcrXn19UjnFJYRFmzFbrO2UlBCCNE85kqkPSdeh6dH2vjaL7e4zFcRCSGEaApnqXFdT0U6WxZjEUKYhLkS6WoV6WipSAshhLl4KtINzCMdGy5tHUII/2eyRLpqj3SU3ZgGW3qkhRDCJBrR2pFdKBVpIYQ5mCyRrrQgCxBktRBpD5JlwoUQwiwqEun6BxtKIi2EMAOTJdKeJcKLK+6S1Q2FEMJEKnqk6x9sKFPfCSHMwFyJdFDV1g4wpkeS1g4hhDCJBirSznIXZ0qcMvWdEMIUzJVIWyyUW4IrBhuCsUy4ZxUsIYQQfs7hSaRrH2zoKYxIRVoIYQbmSqQBlyX47IkYo7VDVjYUQgiTaKAinVMoy4MLIczDhIl0SJWKdHSYJNJCCGEaDczakS2JtBDCREyXSJdbQ6r2SIfayC1yoLX2YVRCCCEapYEFWXLcszDJPNJCCDMwXSJtVKSrDjZ0ujSFZeU+jEoIIUSjNLAgS06RVKSFEOZhukS63BpcY/o7QNo7hBDCDBrqkZZEWghhIqZLpKtXpKNDjZOtzNwhhBAm0ECPdE5hGaE2K6HB1lYMSgghmsd0ibTRI11p+jv3XKN5srqhEEL4P2cpoMBae8U5u9BBrMwhLYQwCdMl0rVNfwfS2iGEEKbgLDGq0UrV+nBuURmxMoe0EMIkTJhI1xxsCMjqhkIIYQaOkjr7owGyi8qkP1oIYRqmS6RrtHZU9EhLIi2EEH7PU5GuQ26RQyrSQgjTMF0iXb0ibbdZCLZapLVDCCHMwFlaf0W6sIw46ZEWQpiE6RLpiunv3AuwKKXcqxvKrB1CCOH36qlIO8td5BU7iJHWDiGESZgukXZZQkC7oPxs4uxZ3VAIIYSfc5aArfZE2vPNYpy0dgghTMJ0iXS51f2VYLUp8CSRFkIIE6inIu1ZjCVGWjuEECZhukTaZXFXKqpNgSc90kIIYQL19EjnFElFWghhLiZMpGtWpKNDgyWRFkIIM6inIp1dKMuDCyHMxXSJ9NnWjqpzScsS4UIIYQKOelo7PIm0VKSFECZhukT6bEX6bCIdHWqjsKwcR7nLR1EJIYRolHp7pN2tHVKRFkKYRIOJtFKqo1JquVJqp1Jqh1Lq8Vq2UUqp15RS+5RSW5VSF7ZMuO7p78CYAs/NMzBF2juEEMLP1dsjXUZIkIXQYGsrByWEEM3TmIq0E3haa90XGAk8opTqW22ba4Ce7st0YIZXo6ykroo0yOqGQgjh9+qrSBeWyUBDIYSpNJhIa62Pa603um/nA7uApGqbjQfmasNqIEYp1d7r0VL79HeeRFoq0kII4eecpXXOI51TVCYDDYUQptKkHmmlVBdgCLCm2kNJwJFKP2dQM9n2itqmv/OsgiWrGwohhB/T2mjLq2fWjthwmUNaCGEeQY3dUCkVAXwKPKG1PtOcgymlpmO0fpCYmEhaWlqT9+EocQKQvmMzx3KNondmoTHI8PsNW7Gc8P1JuKCgoFmvrTVJjN4hMXqHxHiecDmNlWnr6JHOLXLQISa0lYMSQojma1QirZSyYSTR87XWn9WyyVGgY6Wfk933VaG1ngnMBEhJSdGpqalNjZdvliwCoFfXjvS6yHh+blEZP/3ma9p37kHq6K5N3qe3paWl0ZzX1pokRu+QGL1DYjxPON3fJNZVkS6SHmkhhLk0ZtYOBcwGdmmtX65jsy+Be9yzd4wE8rTWx70YZ4WzrR1nBxtG2qVHWggh/J6z1LiuJZEud2nyih0VrXpCCGEGjalIXwzcDWxTSm123/dzoBOA1votYBFwLbAPKALu9XqkbtoSBJagKtPfWS2KKHuQJNJCCOHPPAWQWhLpvGIHWkNcmO/b84QQorEaTKS11t8CqoFtNPCIt4JqkC2sSkUajAGHsrqhEEL4sXoq0tmyqqEQwoRMt7IhALbQKtPfgTEFnlSkhRDCj1X0SNccbOgphMj0d0IIMzFnIh1krzL9HRirG+ZKIi2EEP6rERVpGWwohDATcybStrDaK9KysqEQQqCUGqeU2qOU2qeUeq6Obe5QSu1USu1QSn3QKoF5xrbUsiCLZ2XaGOmRFkKYSKPnkfYrttBaeqSltUMIIZRSVuAN4EqMxbHWKaW+1FrvrLRNT+BnwMVa6xylVNtWCa6e6e+yi6QiLYQwHxNXpKsm0tGhRmuHMe5RCCHOW8OBfVrrA1rrMuBDYHy1baYBb2itcwC01lmtEllFa0fNHumcwjKCgyyE2qytEooQQniDSRNpe5Xp7wBiQoMpd2kKSp0+CkoIIfxCEnCk0s8Z7vsq6wX0Ukp9p5RarZQa1yqR1VORzikqIy4sGGPpAiGEMAfztnacOVblrmh3X11ukaNigRYhhBC1CgJ6AqkYK9GuVEoN0FrnVt5IKTUdmA6QmJjYrCXSKy+t3u74ZvoAq9dvpiS06ppdew+XYNPaJ8uwm2H5d4nROyRG75AYzzJpIl37YEMwJvXvWNtzhBDi/HAUqpwGk933VZYBrNFaO4AflFLpGIn1usobaa1nAjMBUlJSdHOWSK+ytPq6/bAHRo5OhcjEKtv9fdcqOkZaSE0d2eRjnCszLP8uMXqHxOgdEuNZ5mztCLLXHGwYKsuECyEERjLcUynVVSkVDNwJfFltm39iVKNRSiVgtHocaPHI6uuRLiqTxViEEKZjzkTaFlbLPNLGCThXpsATQpzHtNZO4MfAYmAXsFBrvUMp9YJS6kb3ZouB00qpncBy4Fmt9ekWD66+HunCMmJl6jshhMmYtLWj9pUNQSrSQgihtV4ELKp23y8r3dbAU+5L66mjIl3u0uQVO4iTVQ2FECZj3oq0ywHlZ2fo8Ezin1tc5quohBBC1MdZbFSjq83McabYgUsjrR1CCNMxaSLt/lqw0hR4dpuVkCCLrG4ohBD+yllaa3+0ZzGWWKlICyFMxqSJdKhxLasbCiGEeThLau2PzvUk0lKRFkKYjEkT6TDjupY+aRlsKIQQfqquinShcd6WwYZCCLMxZyLtqWjUmAIvWHqkhRDCXzmKISi0xt050tohhDApcybSFRXpqol0dJhUpIUQwm/VUZHOKTQS6Thp7RBCmIxJE+nae6SjQ22ckR5pIYTwT3X0SGcXlRFstRAWbPVBUEII0XwmTaRrr0jHhNrIlURaCCH8Ux0V6dxCB7HhNlS1afGEEMLfmTSRrjn9HRizdhSVlVPmdPkgKCGEEPXyzCNdTXZRmfRHCyFMyaSJdB090rK6oRBC+C9n6dlCSCW5kkgLIUzKpIm0p0e62vR37hNxnszcIYQQ/qeuHunCMhloKIQwJXMm0nVOf+deJlxm7hBCCP9T16wdRQ5iZA5pIYQJmTORrmuwYZi0dgghhN9y1OyRdrk0uUVSkRZCmJM5E+mgEEDV2SMtFWkhhPBDztIaifSZEgcuDTHSIy2EMCFzJtJKGVXpaj3SMaHGiVimwBNCCD9US490jrvwERcurR1CCPMxZyINxshvZ0mVuyLtQSgFeUUy2FAIIfxKuRN0eY1EOrtQlgcXQpiXiRPpsBqtHRaLIspukx5pIYTwN57CR7XBhjmSSAshTMzEiXRojdYOMAYcSmuHEEL4GU8i7Zm+1C3H/Q2iDDYUQpiReRPpIHuNijS4lwmXwYZCCOFf6qpIuxNpmf5OCGFG5k2ka2ntAIgKldYOIYTwO85S47qWwYY2qyIiJMgHQQkhxLlpMJFWSs1RSmUppbbX8XiqUipPKbXZffml98OshS209op0WLAk0kII4W/q6ZGODQtGKeWDoIQQ4tw0pgTwHvA6MLeebb7RWl/vlYgayxYGhadq3G20dsisHUII4VccnkS65qwdMtBQCGFWDVaktdYrgexWiKVpbHZw1laRNlo7XC7tg6CEEELUyll7Ip1b5CBW5pAWQpiUt3qkRymltiilvlJK9fPSPutXR2tHdKgNl4aCMmerhCGEEKIR6kiks4ukIi2EMC9vjO7YCHTWWhcopa4F/gn0rG1DpdR0YDpAYmIiaWlpTT5YQUEBaWlp9MjKIbHoDN9V28fxDKM/evGyb2gT5puxlJ4Y/ZnE6B0So3dIjOeBisGGVXukc4vKiJWp74QQJnXOibTW+kyl24uUUm8qpRK01jUamLXWM4GZACkpKTo1NbXJx0tLSyM1NRXKlkLmUqrvw7Ezk9nb19Nn4FAGJEc3ef/eUBGjH5MYvUNi9A6J8TzgacWrVJF2uTQ5RQ7ipCIthDCpcy7ZKqXaKfdwa6XUcPc+T5/rfhtkCzO+KnS5qtwdHWr02snMHUII4Uc8FWnb2UQ6v8RJuUvLHNJCCNNqsCKtlFoApAIJSqkM4FeADUBr/RZwG/CQUsoJFAN3aq1bfqSfZ3UsZwkEh1Xc7Tkh5xbLzB1CCOE3aumRllUNhRBm12AirbWe2MDjr2NMj9e6bO7k2VFcNZF2V6RldUMhhPAjtfRIZ7sTaRlsKIQwKxOvbOiualSbAi9KWjuEEML/OGr2SHvm/JfBhkIIszJxIl2pIl2J3WbFbrNIIi2EEP6kliXCswuN87QMNhRCmJWJE2l3j7SjqMZDMaHBsrqhEEL4E2cJWEOg0lLgOYXGeTpGFmQRQpiUeRNpT1WjlkVZPKsbCiGE8BPO0hqLseQUlRFkUUSGeGNJAyGEaH3mTaTraO0Ao09aBhsKIYQfcZbUWIwlp6iMmLBgVKUqtRBCmImJE2lPa0ctFelQqUgLIYRfcZbUXB68sIw4aesQQpiYiRNpT0W6lh7pMKlICyGEX3GWVFmMBSCnyCFT3wkhTM3EiXTdPdLRUpEWQgj/4iyt2dpRWCaJtBDC1EycSLsr0p7VsiqJCQum2FFOiaO8lYMSQghRq1paO3KKHDKHtBDC1EycSNc9/V20e1GWM1KVFkII/+ComkhrrckpKiM2THqkhRDmZd5EOqjuwYbRsrqhEEL4l2oV6TMlTspdmjipSAshTMy8ibTFYkzuX8c80gC5kkgLIYR/qNYjXbE8uPRICyFMzLyJNBjtHbVOf2ecmGXmDiGE8BPVKtLZ7lUNY2X6OyGEiZk8kQ6rc/o7kNYOIYTwG9USaU+hQyrSQggzM3kiba9zZUM4+9WhEEIIH6s2j3RFRVoSaSGEiZk8kQ6rdfq7yJAgLEoq0kKI85NSapxSao9Sap9S6rl6trtVKaWVUiktHpSztEpFOsfTIy2DDYUQJmbyRDq01tYOi0URHSqrGwohzj9KKSvwBnAN0BeYqJTqW8t2kcDjwJpWCcxZUmWwYU5RGVaLIsoe1CqHF0KIlmDuRDqo9tYOkNUNhRDnreHAPq31Aa11GfAhML6W7X4LvATU/FrP28qd4HJWG2zoIDbMhlKqxQ8vhBAtxdylAFsYFGTW+lB0WLBMfyeEOB8lAUcq/ZwBjKi8gVLqQqCj1vo/Sqln69qRUmo6MB0gMTGRtLS0JgdTUFDAN8u/Zgyw/1AGR9z72HuohGBczdqntxUUFPhFHPWRGL1DYvQOifEskyfStU9/BxATapPBhkIIUY1SygK8DExtaFut9UxgJkBKSopOTU1t8vHS0tIYM2wAfAvde/en+whjHzP2fE9yGKSmjmryPr0tLS2N5ry21iQxeofE6B0S41nmbu2whUlrhxBCVHUU6Fjp52T3fR6RQH8gTSl1EBgJfNmiAw49g8Kr9UjL8uBCCLMzeSJtr3WwIRhzSUtrhxDiPLQO6KmU6qqUCgbuBL70PKi1ztNaJ2itu2ituwCrgRu11utbLKKKRLryrB0OWR5cCGF6Jk+ka5/+DozWjrxiBy6XbuWghBDCd7TWTuDHwGJgF7BQa71DKfWCUupGnwRVrSKttSansIwYmUNaCGFyAdAjXQRaQ7WR39FhwWgN+aVOokPl60MhxPlDa70IWFTtvl/WsW1qiwfkSaRtoQAUlDpxujRxkkgLIUzO3BXpIDtoF5TXHFToSZ7zZC5pIYTwLWepce2uSOcUupcHl9YOIYTJmTuRtoUZ17UMOIzxLBNeLDN3CCGET1Xrkc72rGoogw2FECZn8kTa+Jqw1kTafYKW1Q2FEMLHHFV7pGV5cCFEoDB5Iu2pSNecuaOitUNm7hBCCN+qqEgbxY+cQk9FWhJpIYS5mTyRdk+lVEtFOtpTkZZEWgghfKt6j7T7m0IZbCiEMDuTJ9LuinQtU+CdHWwoPdJCCOFT1XqkcwrLsFoUkXZzTxwlhBANJtJKqTlKqSyl1PY6HldKqdeUUvuUUluVUhd6P8w6VPRI12ztCAmyEmqzSmuHEEL4WrV5pLOLyogJtWGxqHqeJIQQ/q8xFen3gHH1PH4N0NN9mQ7MOPewGimo7sGG4F7dUAYbCiGEb1WrSOcWlclAQyFEQGgwkdZarwSy69lkPDBXG1YDMUqp9t4KsF71zNoBRnuH9EgLIYSPVfRIu6e/KyyTqe+EEAHBGz3SScCRSj9nuO9reQ0k0jFhNmntEEIIX3OWgDUYLMZ/OblFDpmxQwgREFp1pIdSajpG+weJiYmkpaU1eR8FBQUVzwsuzeEiIH3HZo7l1iyCOwpLyCx0Nes456JyjP5KYvQOidE7JMYA5yytqEaDUZEe3DHGd/EIIYSXeCORPgp0rPRzsvu+GrTWM4GZACkpKTo1NbXJB0tLS6PieSV58D306tqRXhfV3NdXp7aSkZ5Fc45zLqrE6KckRu+QGL1DYgxwjuKKgYZaa3KKyoiRirQQIgB4o7XjS+Ae9+wdI4E8rfVxL+y3YRXT38lgQyGE8FvO0orB4YVl5TjKNXHh0iMthDC/BivSSqkFQCqQoJTKAH4F2AC01m8Bi4BrgX1AEXBvSwVbg9UGlqA6e6SjQm2UOl2UOMqx26ytFpYQQohKnCVnF2Nxr2ooFWkhRCBoMJHWWk9s4HENPOK1iJoqKLTewYZgLBMuibQQQvhIpR7pHPciWbKqoRAiEJh7ZUMwZu6oK5EONU7U0t4hhBA+5DzbI53trkjLPNJCiEAQ0Il0xTLhMgWeEEL4jrO0YrpST0Va5pEWQgSCAEikw2pdIhzOtnbkuk/cQgghfKBKj7RR2IiTirQQIgAEQCJtb7AiLasbCiGED1XrkbYoiLJLRVoIYX4BkEiHGdWOWngq0mckkRZCCN+pNI+0Zw5pi0X5OCghhDh3AZBIh9bZ2hEREoTVomSwoRBC+FLlinShQ/qjhRABw/yJdFDdrR1KKaJDbeQWS4+0EEL4jLOkIpHOLiwjVqa+E0IECPMn0vUMNgSICZXVDYUQwqeq9UjL1HdCiEARAIl0KDhq75EGY3VDmf5OCCF8yFm1R1paO4QQgSIAEumwOls7wBhwKIm0EEL4hnKVg8sJQXa01kaPtFSkhRABIgASabu0dgghhJ9S2n3+tdkpKiunrNwly4MLIQJGACTSYeByQLmz1oejpbVDCCF8xuJyD/YOsp9dHlwSaSFEgAiARNpYdhZnHYuyhAVzpsRBuUu3YlBCCCEALC53ISMopOLbQWntEEIECvMn0u6R4HX1SceE2tAa8kukKi2EEK3NWl6pIl3kqUjLYEMhRGAwfyJtCzOu6+iT9qxuKO0dQgjR+iq3duR4WjukIi2ECBABkEi7WzvqmAIvOtRIpGXAoRBCtL4qibS7Ii2DDYUQgSIAEunGVaRzpSIthBCtrnKPdE5hGUoZ8/sLIUQgCIBEuv4e6ehQo/KRWyTLhAshRGurWpF2EBNqw2pRvg1KCCG8JAASaXdFuq5ZO9yVjzNSkRZCiFZ3NpEOIbuoTKa+E0IElABIpD090vUn0tIjLYQQra+itcMWSk5hmQw0FEIEFPMn0kH1J9LBQRbCg63SIy2EED5QuSKdU+SQirQQIqCYP5GuqEjXs0x4WLBMfyeEED5Qffo7mUNaCBFIAiiRrn36OzBGiEtrhxBCtD5PIq2tIeQUlREnrR1CiAASAIl0/dPfgbG6YV6xzNohhBCtzdMjXYyNUqeLGGntEEIEEPMn0kEhgKqzRxqMuaSltUMIIVqfpyKdXWJMeRcXLq0dQojAYf5EWimjvaOO6e/AmLlDWjuEtz21cDN/WFNMVn7dbUVCnO8srjKw2MgtcQHIYEMhREAxfyINRiJdT0U6OsxGbrEDrXUrBiUC2bHcYj7fdJQ9OS5ufmMV6Zn5vg5JCL9kcZVBkJ3sQqMyLdPfCSECSWAk0kH1J9IxocGUOV2UOFytGJQIZJ9uyEBreHhwCGXlLm6dsYpv957ydVhC+B2LywE2Oznu1WWlIi2ECCSBkUjbQusdbOhZlEX6pIU3uFyajzdkMKpbPMPbBfHPRy6mQ3QoU99dy8J1R3wdnhAopcYppfYopfYppZ6r5fGnlFI7lVJblVJLlVKdWyoWT0U6x1ORlunvhBABJIAS6br7VGPcJ+5cmblDeMGaH7I5nF3EHcOSAUiKCeXjh0Yxqns8P/l0K3/6725cLmkjEr6hlLICbwDXAH2BiUqpvtU22wSkaK0HAp8Af2qpeCwuh3t5cAdKnS1sCCFEIAiQRDqswenvQJYJF96xcP0RIkOCGNevfcV9UXYbc6YOY+LwjryZtp/HPtxEiaPch1GK89hwYJ/W+oDWugz4EBhfeQOt9XKtteekuRpIbqlgrOVGRTq3qIzoUBtB1sD4b0cIIaCRiXQjviacqpQ6qZTa7L484P1Q62GzNzjYEKS1Q5y7MyUOFm07zo2DOxAabK3ymM1q4cWbB/DcNX3499bjTH5nDacLSn0UqTiPJQGVe4wy3PfV5X7gq5YKxmjtCCG7sEz6o4UQASeooQ0qfU14JcYJeZ1S6kut9c5qm36ktf5xC8TYMFsYFJ2u8+GKHmmpSItz9K8txyh1urgjpWOtjyuleHBsdzrFhfHkR5u5ZcYq5kwdRvc2Ea0cqRANU0rdBaQAY+t4fDowHSAxMZG0tLQmH2OAo4TcAis/5GZiLadZ+2hpBQUFfhlXZRKjd0iM3iExntVgIk2lrwkBlFKerwmrJ9K+08D0d56VtKRHWpyrhesz6J0YycDk6Hq3u3ZAe9pF25n2/npueXMVb989lJHd4lspSnGeOwpU/qSX7L6vCqXUFcDzwFitda1fnWitZwIzAVJSUnRqamqTgzmzoZyohA7o7Ag6R9tJTR3W5H20tLS0NJrz2lqTxOgdEqN3SIxnNaa1o7FfE97qHgH+iVKq9nJdS2lg+rvwYCtBFiWtHeKc7DmRz5YjudwxrCNKqQa3v7BTLJ8/fDEJEcHcPXsNn2/KaIUohWAd0FMp1VUpFQzcCXxZeQOl1BDgbeBGrXVWSwZTMWtHUZnMIS2ECDiNqUg3xr+ABVrrUqXUj4D3gcuqb+SNrwlrK9V3P5VHUn4mOz55idMJI2p9XmiQZue+Q6SFnGjyMb0Ro7+RGJtuwa5SrAraFB0kLe0Q0LgYnxqoeX2T4smPtrBiw07Gd7c1KhH3Fn97H2sjMXqP1tqplPoxsBiwAnO01juUUi8A67XWXwJ/BiKAj91/i4e11je2RDyeHumcojLiJJEWQgSYxiTSDX5NqLWu3KD8DnVMpeSNrwlrLdUP6gofHWLA9hdh8GQY9wewV/3qvc2GNMJio0hNvbDJx/RKjH5GYmyaMqeLp75ZylX92nHjVUMr7m9sjFde6uJnn23j040ZWCLb8odbBxASZG3wed7gT+9jXSRG79JaLwIWVbvvl5VuX9FasVhcDpzWEEocroqpSIUQIlA0prWjMV8Ttq/0443ALu+F2AixneGBpXDJs7DlQ5hxMRxYUWWTmFCbDDYUzbZsdybZhWV1DjJsSHCQhb/cPpCnruzFZ5uOcs/steQWSc++CHwWVxml2kig42TWDiFEgGkwkdZaOwHP14S7gIWerwmVUp6vAh9TSu1QSm0BHgOmtlTAdQoKhst+Afd/DUF2mHsjfPVTKDOmSo0OtUmPtGi2j9YdoV2UnUt6tWn2PpRSPHZ5T16ZMJhNh3O5ZcYqDp0u9GKUQvgfi6uMYnciLT3SQohA06ge6UZ8Tfgz4GfeDa2ZkofCj1bC0hdgzQzY/imEJfD7M6UUlGl4OwqU1Zgyr99NMHAC2KN8HbXwYyfySliRfpKHUrtjtZx7b/NNQ5LoEBPK9HnrufnNVXwwbQR92snfoAhMFpeDYpfxX43MIy2ECDSBucRUcBhc80eY8i/olgoJPcmzJ3NUx0NEOwiLg5JcWPQMvHwB/OdpyKqlG8VRDPuXw9e/gtlXw6JnIU9mXjjffLoxA5eG24d6bzKa4V3j+Oyhi7BZFXe9s5aDp6QyLQKQqxyLdlLocrd2hEuPtBAisHhr1g7/1PUS4wIsXpLOK0v2sn/itWerihkbYN07sHGecd15NAy5C/KPwYE0OLwGykvBYoN2A2D9HFj/LgyZDKOfhNguPntponVorfl4/RFGdI2jS0K4V/fdrU0E/7h/BHe8/T2T31nDxw+OokNMqFePIYRPOY3pqQvKjf9qYqQiLUS9HA4HERER7NrVukPNmio6OtpvY7Tb7SQnJ7fa8QI7ka7Es7rhmWLH2T695KHG5arfwaZ5sH42/PNB47HEATB8mlHR7jQKQiIg9zB8+4qx7cZ5MGgijHkK4rv75DWJlrf2h2wOni7i0ct6tsj+eyZGMve+EUyatZq73lnDwgdHkRAR0iLHEqLVOUsAKCg3ZqiJCZWKtBD1ycjIIDExkeTk5FadJrWp8vPziYyM9HUYNWitOX36NBkZrdc9cN4k0p5pl3IrJ9Ie4fEw+gm46FE4tgliOkNELYPKYjrB9S/DJc/Ad6/BhndhywfQ40rodRX0vBpimvj1v7MMSs9ASR6Ul0F0RyNpF35h4foMIkKCuGZAuxY7xoDkaObcO4y7Z6/hntlrWTB9ZMUHPyFMzZ1In3FYiQ61EWQNzG5CIbylpKSEpKQkv06i/ZlSivj4eE6ePNlqxzx/EulQI3mud+YOixWSUxreWVQHowd79JOw+k3Y8TnsXQw8DW370i2kD3SxQVxXo4qdexhyDkHuIeN2/nEjcS45A85aVmSMaAdx3dyXrsZ1dEfjuJHtjDh9pbQATu42esXb9IGEnt6PR2vI3E5Y4WHv7reJ8kscLNp2nJuGdCAsuGX/qQzrEsfbd6fwwPvruPfdtcy7fwThIS3/z/O7fafYeewM917cRZIc4X3uRDrPYSVW5pAWolEkiT43rf3+nTeJdJS7wufVuXsjE+HK38AVv4ZTe41keu//SD74Bbz3Wc3tIxKNqnbbC8AeY8wWEhJtLB5jjwJLkJFsnz4A2Qdg3xIoqLYSo7JCZHuITjISa3sMBIdDcIRxHRJh3A5PgKgk49LUCrerHApPGQl/7iHI3HH2kvND1W1t4dB+EHQYYlzaDTAGe2oNaGMbrUEp4wOCzV73cXMOwtaPYdtCOJXOcIBTH8PIR6DnVWBp3UTv31uPU+wob/bc0U01tlcbXrtzCI98sJEfzdvAO1NSsNta5kOTo9zFX/+Xzlsr9gOwdHcmr0+6UNpKhHe5e6RzHVaZ+k4IEZDOm0Ta09rRInNJKwVtehmXix7luyWLGNPBCYVZRptITGej5cPWjIFkpQVGgnnmqFEFPnPs7O3jW6E0H8oKwFFU9z5Coo2kOzoJQiIBRd+sLDg11/MCjOfnnzAuBZmgyyu9PgvEdTcS5sGTILGfsb+s3UYrzLFNRn+5u/pU9/tkMQZoJvSGNr2NinZ8DzixBbYuhCNrjO06XQTXP8T+nZvpfmoJLJhgbDfiQeP4wd4d9FeXheuP0LNtBIM7xlR9oNxpfMjIO0LiiTRYuc748BHVASI7QFT7sx9ymvjJ+JoB7fnTbYN45uMtPLpgE29OvhBbbZViraGs0LiExYO18f+Uj2QX8diHm9h0OJeJwzsxpGMM//fFdm78+7fMuGsog6q/XiGay31OyC5VxCZIIi2EGeTm5vLBBx/w8MMPN+l51157LR988AExMTEtE5ifOn8S6YqK9Lkl0vklDoIsFkKD664UlgeFQd/UczpOhZAIaNffuNTHVW4kw2WFRvJdmAV5R42k25N8nzlqtJgAEUWF4DrhrhxjLGIT2Q7a9jWuI9udrXwn9DaqzNV1GAKDJ7pftNNo+cjcAS7Pe6zOJpLaZbS1nNwNJ9ONarur0u+izQVw+a9gwG1G1R44UpBG98l/hZ1fGC00i56BZb+DXlcbM6mgz8aPBmuw0QoT38NI/OO6Vv3wUpwLp/fByT1wKt2ornuq9+FtIaKtcTssgSMnskjOWM7kvjbU4sVG4px//OyHGfcHjQsAdtfxOwkKNWJIGgodh0PycEjoVbWy7ig2PogcWQNH1kLmdm7TcFWs5uQ+J6f+ZKddbARKWY1tPR+cSvOpqPiHREHni6HbWOg61vjGo44E/r/bj/OTT7aiNbw+aQjXD+wAQN8OUfxo3gZuf/t7fndT/+ZX4V0uKM0zPrw19RsErY3xAkWnoSjHuC4rMF6LsmD8PVmMn11OKM6Bomwozja2L84xZtlJHg7dL4UOFzbpA4ZoAY5KibTM2CGEKeTm5vLmm2/WSKSdTidBQXWfUxctWlTnY4HsvPlfxtPa0dyKdImjnLdXHODNtH0AjOoez2V92nJp77Z0jKslyWxtFqtRbQ6JhEggoUe9m69NSyM1NdV7x7cGNS7h9yh3GJX2U+lG4pzYv/bkz2ozkuv+txqJ5uo34OC37gc9ibr7eY4iKDpV6ckKopONDwW5h41Ku4fFZhzXWQIFWVWTeqAj8PdgYB9wKMz4UBHVwUhYYzoa+43uyJr044y44hajLafghPtDi/uSf9x4fbv/bcz0AkYbT1KKsaz9sc1wYquRFIKR/CcPB2swUS4nmSdy2Hwily4hwfRpG4YKDoNg9+84JMK4toVB5nY4sALSvzL2E97WmPYxpiM4iul1eD/lJ+exJyOL0Oxc5ts1PRNCsa8FVjvB5aS/drEiSrFL29j/hZ1Vq5MY3q8XQZEJRszOUvcHtSIjofd8aCs6DYUnq15rlxFnu/7QbqDR7tN+oPFhyeWE7P1GK9Tpfe7rvYw6dRBW5p99L5oiyA6hccb88ABpf4C0F41kvusYY+adrmOND0lWm/G7twa3eqvQecldkT5dYuECmUNaiCb5zb92sPPYGa/us2+HKH51Q796t3nuuefYv38/gwcPxmazYbfbiY2NZffu3aSnp3PTTTdx6NAhysrKePzxx5k+fToAXbp0Yf369RQUFHDNNdcwevRoVq1aRVJSEl988QWhobV/Kz9r1ixmzpxJWVkZPXr0YN68eYSFhZGZmcmDDz7IgQMHAJgxYwYXXXQRc+fO5S9/+QtKKQYOHMi8efO8+h411XmTSNusFiJCgppVkU7bk8Wvv9zBwdNFXDegPW2jQli+O4tffrED2EHPthFc6k6qU7rEej/4QGS1GQMVExo5rZxS0GmEcalPyRkjUTvtuewzEtoeV7qP18toK4npfLZaqbUx+LPwJBSexJmfxeOf7aVNUld+PekKI5Gso8JbnJF2tlof06miml6F1kYcR9ZCxlo4sg4y1hlJ5kWPuavVw4xEr5IeWvPJf3fz0IoDPNS3Oz8d16f+1557GH5YaSTVP6w0KrW2UGLKrWRmhqCcQfSIiqR9XDQWa5CR/FuCjA9hliCCyh30t50m+dRhrCc3EbSiloGwHtZgI4kPi4fwNsaA2I4jjNdgjzF6649vhc3zjaoyGMeqnihHd4T4HmTHDaV9jwHG/ipfgsPP9ttrl3Fbu4yYQ+MgNLbmtyVF2fDDCti/DPanGR9kaqOsxuu49Gdw8eP1v7eiedw90nnOIJlDWgiT+OMf/8j27dvZvHkzaWlpXHfddWzfvp2uXbsCMGfOHGw2G0FBQQwbNoxbb72V+Pj4KvvYu3cvCxYsYNasWdxxxx18+umn3HXXXbUe75ZbbmHatGkA/OIXv2D27Nk8+uijPPbYY4wdO5bPP/+c8vJyCgoK2LFjB7/73e9YtWoVCQkJZGdnt+yb0QjnTSINxlzSucWNH2x4NLeY3/5rJ//dcYJubcL5x/0jGN3TSHZ+dUM/DpwsYNnuLNL2nOTd735g5soDRNqD6BOjORlxhNTebWkTKYO3WpU96uzAx8ZSCkJjjEtCT5buOMF/imzMvijFuO9cKXX2Q8OQyU14muK5cX0oKHEyI20/ESFBPHJpPd80xHQyFhQacvZk9cmGDH7+2RYi7MH8ddIgLujTtv5jArHAv7ce4+cfb6BjSBF/uqEL/Tq2NZJaW6jRstLYlgmXyxg4e2ILnNhuPD++h/FexHWvSIL3pKXR3lvfkITFQb+bjYvWxvEPf29U0MvL3Bfn2dvtB3nnuKImd0W6FBtxMthQiCZpqHLcWoYPH16RRAO89tprfPrpp1gsFo4cOcLevXtrJNJdu3Zl8ODBAAwdOpSDBw/Wuf/t27fzi1/8gtzcXAoKCrj66qsBWLZsGXPnGmO5rFYr0dHRzJ07l9tvv52EBCMXi4uL8+IrbZ7zKpGOCbNxphGtHWVOF+98e4C/L92HRvPs1b15YExXQoKq9kV3axNBtzYRPDCmGwWlTr7de4q0PVn8d2sGz36yFYBBydGk9m7LZX3aMiApGoul9sqm8B8frz9C28gQxvaqZS7xVqaU4rfj+1NQ6uTPi/cQaQ/inlFdGnxeQamTX/5zO59tOkqfOAvvPziGxKh6Zkyp5vqBHejRNoIfzdvATQtP86sb2jF5RJumTytksRhtRgk9jPac1qaUsWCSLJrkG5USaZn+TghzCg8/O8A/LS2NJUuWsGTJEhITE0lNTaWkpOZEAyEhZ4uIVquV4uK6v+GcOnUq//znPxk0aBDvvfceaWlpXo2/pZ1XiXR0qK3B1o5V+07xf19sZ//JQq7qm8gvb+hLcmzDPdARIUGM69+Ocf3bcXXcadr0upDlu7NYvieL15bt5dWle0mICCG1dxuuH9ie1N71VwaFb2SdKWH5npNMv6Sb38yrbLEo/nL7IApLy/nlFzsIDw7i1qF1L3+6/Wgejy7YxKHThTx5RS8GWDOalER79GkXxZePjObxjzbxi39uZ2tGLi+M7+/1KflKHOU4XbrhDYX5eBJpbZPBhkKYRGRkJPn5+bU+lpeXR2xsLGFhYezevZvVq1ef8/Hy8/Np3749DoeD+fPnk5SUBMDll1/OjBkzeOKJJypaOy677DJuvvlmnnrqKeLj48nOzvZ5Vfq8SqRjwmykZxbU+tiJvBJ+v2gX/9pyjM7xYbw7dRiXNvA1eF2UUvRPiqZ/UjSPXt6T7MIyVqRnsWz3Sb7emcknGzKYelEXnr/ugtqnNhM+8+nGo5S7NLfXk6j6gs1q4fVJQ7j//XU8+8kWwt0f3CrTWvP+qoO8uGg3ceHBfDBtJCO7xZOWdrTZx40OszF7yjBeWZLO35ftY8+JfGbcNZQOMc2YyrFSnHsy81mZfpKV6adYezAbm3IxoXAnk0Z0okdbWdkzYLh7pKW1QwjziI+P5+KLL6Z///6EhoaSmJhY8di4ceN46623SElJ4YILLmDkyJHnfLzf/va3jBgxgjZt2jBixIiKJP7VV19l+vTpzJ49G6vVyowZMxg1ahTPP/88Y8eOxWq1MmTIEN57771zjuFcnFeJdHRocI1ZOxzlLt5fdZC/fZ2Ow6V54oqePDi2u1erbnHhwdw8JJmbhyTjKHfxh0W7mfPdD+w5kc8bky887/6DKXdp5q85xKmCMtpGhtAmMqTiuk1kSI0Wmtaitebj9UcY1iWWbm38L5mz26zMvDuFu2av4bEFm3hnSgqXuNtPcovKePaTrXy9M5PL+7Tlz7cP8trfldWiePqq3vRPiubphVu44e/f8sbkCxnZLb7hJ7vlFJbxzb5TrEw/yTd7T5J5xkiweiVGcPfIzmzbd5h5qw8y57sfGN41jskjOjGufzuf/S0IL3FXpEsIlsGGQpjIBx98UOv9ISEhfPXVV+Tn5xMZGVnlMU8fdEJCAtu3b6+4/5lnnqn3WA899BAPPfRQjfsTExP54osvatw/ZcoUpkyZ0tBLaDXnWSJtI6/IgdYapRRrDpzml1/sYE9mPpf2bsOvb+xH5/iWXezDZrXwyxv60q9DFD/7fBs3/P1bZt4zlH4dolv0uP4ir8jBox9uYmX6SZSqNA10JdGhNtpGhmArL+afJzbRNspOm4gQ2kaFVLq2ExUa5NWlQNcfyuHAqUIeSvXfftrwkCDemzqcO2etZvq89fzjfmMWk8cWbOJkQSn/d31f7ru4S4sskXp1v3Z0fySC6fPWM/mdNTx/7QXcW8exHOUuNh3O5Zu9J1mZfpKtR/PQ2vjdju6ZwNiebRjTK4H20UZlOy0ti/4po/hkQwYfrDnM4x9uJi48mNuGJjNxeCe6JrTOIjzCy9wV6TJsFYtiCSFEIDmvEumYMBtl5S6OZBfzypJ0Ptt0lKSYUGbePZQr+ya26vrstw5NpmeiMZjr1hmr+NNtg7hxUIdWO74vpGfmM23ueo7lFvPizQO4IyWZ04VlnMwvJSu/xLg+U8rJAuN639EiNhzOIetMKaVOV439JcWE8pfbBzGqe+Mro/VZuO4I4cFWrh3Q3iv7aynRYTbm3jecCW9/zz1z1lLqdJEcG8qnD13EwOSYFj12j7YRfPHIxTy9cAsv/HsnWzNy+cMtAwkNtnIku4gV6Ubi/P3+0+SXOrFaFIM7xvDE5b24pFcCA5NjsNYx4DYhIoQHx3Zn+phufLf/FB+sOczsb43ZcC7uEc+k4Z25sm8iwUHSDmUalzzLj7b3I+xkkLSxCXGee+SRR/juu++q3Pf4449z7733+igi7zi/Emn3oixXvbKCcpfmkUu78+NLe9a7SmFLGpgcw5c/Hs3D8zfw2IJN7DiWx0+u7lNnomFm/91+nKcWGr29H04fydDOxuCAxCi7eyBczYp8mnvRGK01+aXOikTbk3R/sOYwk95ZzY8u6c5TV/Y6pwSroNTJf7Yd54aBHQgP8f9/Fm0iQ5j3wAjufXctfdtH8dub+hNpb52KX6Tdxlt3DeWN5ft4eUl6RbX5h1OFgPEB5/pB7bmkZxsu6pFAdGjT4rJYFGN6tmFMzzZknSlh4fojLFh7hEc+2EhCRDC3p3Rk4rBOdIr3g4WQRP2UIs9pPe/a14QQNb3xxhu+DqFF+H/G4EWeFQiHdYnjNzf284s+2DaRIcx/YCS//fdO3l5xgJ3HzvD3iUMCpp/Q5dK8siSd15btY1DHGN6+ayjtops2g4RSiii7jSi7je6VfmeTRnTit//exVsr9vPdvlO8eufgZv9OF209TlFZOXcM869BhvVJignlf0+O9cmxLRbFo5f3pH9yNC/+ZxdJcaHcPbIzl/RqQ/c24V77dqdtlJ0fX9aTh1J7sHLvSeavPszbK/bz1or9jOnZhknDO3HFBW39ZoYVUVNBGTJjhxAiYJ1XifRF3eNZ8WwqneLCWrWNoyHBQRZ+e1N/+nWI4v++2M74N75j5t0p9G4X2fCT/diZEgdPfriZpbuzuCMl2etTp4UFB/GHWwYwtlcbnvtsK9e99i2/uqEvE4Z1bPLv96P1R+jeJpwLO8nKlE1xaW9jRc+WZrWoimMdzyvmo3VH+GjdER78xwbaRoYwYVhHpl7UhfgIWQDJ3+Q7NN2kP1oIEaDOqzKOUorO8d6rlnnbncM78eH0URSXlXPzm9/x3+3HfR1Ss+3LKuCmN75jRfpJXhjfj5duHej1+Yc9xvVvx+InLuHCzjE899k2HvzHBnIKG7+C5b6sAjYcyuGOlKYn4KL1tY8O5YkrevHNTy5l1j0p9OsQxVsr9uMol7mo/VFBmSZWWjuEEAHqvEqkzWBo51j+9ehoeiVG8uA/NvLX/+3BZbLFKpbszOSmN74jr8jBPx4YwT2jWmYWicoSo+zMu28Ez197Act2ZzHu1ZV8t+9Uo5778YYjWC2Kmy9MatEYhXcFWS1c2TeRd+8dzpqfX9HkliHROgocWlo7hBABSxJpP5QYZeejH43kjpRk/r5sH9PmrudMScNLm/uay6V5beleHpi7ni4JYXz56OgmzTV8riwWxbRLuvH5wxcTERLEXbPX8IdFuyirZcYPD0e5i083HOWyPm1pGymJmFnJYDb/VOIop7Rcfj9CBLKICN+PN/MlSaT9VEiQlZduHchvx/djRfpJbnrjO/Zl1b4qoz8oKHXy8PyNvPx1OjcPSeKTBy8i6RxWvzsX/ZOi+fejY5g0vBNvrzzAzW/W/d6l7TnJqYJS7kjp2MpRChH4couMAoBUpIUQgeq8GmxoNkop7h7VhV6JkTw8fyM3vfEdr0wYzBV9Ext8bqmznJP5pRWXLM9t9xzN+Tkl7LUcYGByNP2Sook4hynfDp4qZNrc9ew/WcAvrruA+0d39XmvcWiwld/fbAxE/OmnW7n+79/wf9f3ZdLwTlViW7j+CAkRIaT2buPDaIUITNnusQqxMthQiKb76jk4sc27+2w3AK75Y72bPPfcc3Ts2JFHHnkEgF//+tcEBQWxfPlycnJycDgcPP/889x5550NHq6goIDx48dXPO93v/sd48ePB2Du3Ln85S9/QSnFwIEDmTdvHpmZmTz44IMcOHAAgBkzZnDRRRed44tuWZJIm8CIbvH869HR/GjeBh6Yu54nrujJsC5xFfMpV06UPdfVl0IHUAriwoJpExnCyVwXaxbtqri/R5sIBiRHMyg5hgHJ0fRtH9WowYFpe7J4bMEmLBbF3PtGMLpngtdf/7m4ql87BneM4emPt/D859tJ23OSl24dSFx4MFn5JSzbncUDo7vKYhFCtIDcInciLa0dQpjGhAkTeOKJJyoS6YULF7J48WIee+wxoqKiOHXqFMOHD2fChAkNFs3sdjuff/55xfNGjhzJjTfeyM6dO/nd737HqlWrSEhIIDs7G4DHHnuMsWPH8vnnn1NeXk5Bgf9+E+8hibRJdIgJ5eMHR/Gzz7bxypK9VR6z2yy0jbTTJjKEHm0iuKh7PG0iQmgTeXY57bZRIcSFB1ckjGlpafQbOortR/PYkpHLtow8Vqaf4rONRwEIsih6JUYyMDmagckxDEyOpne7yIrna615a8UB/rR4N70TI5l1T0rFPN3+pm2UnffvHc67qw7y0le7GffKSv56xyB2HjtDuUtzu7R1CNEisj2JtLR2CNF0DVSOW8qQIUPIysri2LFjnDx5ktjYWNq1a8eTTz7JypUrsVgsHD9+nMzMTNq1a1fvvrTW/PznP6943tGjR8nMzGTZsmXcfvvtJCQYxbe4OGORtmXLljF37lwArFYr0dE1F2vzN5JIm4jdZuXlOwZx57COaIzFXNpGhhAREtSsVoo2kSFc2qctl/Yx5gHWWnPiTAlbjuSx7WguWzPy+Gr7CT5cdwQw5ru+oH0Ug5KjyTxTwuIdmVw3sD1/vm0gYcH+/adksSjuH92VUd3iefzDTdw9ey2RIUEM7RxLj7bn90AJIVqKZxrK2HBp7RDCTG6//XY++eQTTpw4wYQJE5g/fz4nT55kw4YN2Gw2OnfuTElJSYP7qf68Ll26NOp5ZuLf2Y+oQSnFiBaaCUMpRfvoUNpHhzKuv/EpU2vN4ewitmbksTXDSK4/3ZBBsaOcn47rw4Nju/m8H7op+naI4l+PjubFRbuY+/0h7hrZydchCRGwcmSwoRCmNGHCBKZNm8apU6dYsWIFCxcupG3btthsNpYvX87hw4cbtZ+8vLwqzzt06BAAl112GTfffDNPPfUU8fHxZGdnExcXx+WXX86MGTN44oknKlo7/L0qLYm0qJdnEZvO8eHcMKgDAOUuTbGj/JwGKPqS3WblhfH9efSynrSJlJXwhGgpk0Z0IqLgsIxBEMJk+vXrR35+PklJSbRv357Jkydzww03MGDAAFJSUujVq1ej9lP9eX369KnY//PPP8/YsWOxWq0MGTKE9957j1dffZXp06cze/ZsrFYrM2bMYNSoUS35Us9ZozIhpdQ44FXACryjtf5jtcdDgLnAUOA0MEFrfdC7oQp/YbUo0ybRlUkSLUTLSogIoVt0y6xoKoRoWdu2nZ0xJCEhge+//77i5/z8fCIjIwHqHRBY/XmVTZkyhSlTplS5LzExkS+++OJcwm51DZYJlFJW4A3gGqAvMFEp1bfaZvcDOVrrHsDfgJe8HagQQgghhBD+pDFlxeHAPq31AQCl1IfAeGBnpW3GA7923/4EeF0ppbTW5lrbWgghhBBCNMm2bdu4++67q9wXEhLCmjVrfBRR62lMIp0EHKn0cwYwoq5ttNZOpVQeEA+c8kaQQgghhBDCPw0YMIDNmzf7OgyfaNVGV6XUdGA6GH0waWlpTd5HQUFBs57XmiRG75AYvUNi9A4zxCiEMD/5Mv/ctPb715hE+ihQecWKZPd9tW2ToZQKAqIxBh1WobWeCcwESElJ0ampqU0OOC0tjeY8rzVJjN4hMXqHxOgdZohRCGFudrudvLw8IiMjTTW1rL/QWnP69GnsdnurHbMxifQ6oKdSqitGwnwnMKnaNl8CU4DvgduAZdIfLYQQQgjReMnJyWzZssXvl8YuKSlp1WS1Kex2O8nJyRVzVre0BhNpd8/zj4HFGNPfzdFa71BKvQCs11p/CcwG5iml9gHZGMm2EEIIIYRoJJvNRkFBASkpKb4OpV5paWkMGTLE12H4hUb1SGutFwGLqt33y0q3S4DbvRuaEEIIIYQQ/kuWmxJCCCGEEKIZJJEWQgghhBCiGZSvxgQqpU4CzekET8D/56eWGL1DYvQOidE7qsfYWWvdxlfBtDY5Z/ucxOgdEqN3mDHGFjln+yyRbi6l1HqttV934UuM3iExeofE6B1miNEfmeF9kxi9Q2L0DonRO1orRmntEEIIIYQQohkkkRZCCCGEEKIZzJhIz/R1AI0gMXqHxOgdEqN3mCFGf2SG901i9A6J0TskRu9olRhN1yMthBBCCCGEPzBjRVoIIYQQQgjf01qb5gKMA/YA+4DnWvhYHYHlwE5gB/C4+/5fA0eBze7LtZWe8zN3bHuAqxuKG+gKrHHf/xEQ3Iw4DwLb3LGsd98XB3wN7HVfx7rvV8Br7uNtBS6stJ8p7u33AlMq3T/Uvf997ueqJsbXu9J7tRk4Azzh6/cRmANkAdsr3dfi71tdx2hCjH8Gdrvj+ByIcd/fBSiu9H6+1dxY6nu9jYyxxX+3QIj7533ux7s0McaPKsV3ENjsy/cxUC91/V5b6Fhyzq7n77QJ8fnlOdv9PL8+b9cRn5yzdWCfs31+om3CP24rsB/oBgQDW4C+LXi89p43EYgE0oG+7j+4Z2rZvq87phD3H9J+d8x1xg0sBO50334LeKgZcR4EEqrd9yfPHzbwHPCS+/a1wFfuP5aRwJpKf1QH3Nex7tueP6y17m2V+7nXnOPv8ATQ2dfvI3AJcCFV/6G2+PtW1zGaEONVQJD79kuVYuxSebtq+2lSLHW93ibE2OK/W+Bh3CdM4E7go6bEWO3xvwK/9OX7GIiX+n6vLXQ8OWfX83d6Dr9Dvzhnu7f16/N2HfHJOVsH9jnbTK0dw4F9WusDWusy4ENgfEsdTGt9XGu90X07H9gFJNXzlPHAh1rrUq31DxifaIbXFbdSSgGXAZ+4n/8+cJOXwh/v3l/1/Y4H5mrDaiBGKdUeuBr4WmudrbXOwfiENs79WJTWerU2/srmnmOMlwP7tdb1LerQKu+j1nolkF3LsVv6favrGI2KUWv9P6210/3jaiC5vtfZzFjqer2NirEe3vzdVo79E+By9/ZNitH9nDuABfUF3tLvY4CSc3bjyTm7Efz9vC3n7PPznG2mRDoJOFLp5wzqP0l6jVKqCzAE4+sIgB8rpbYqpeYopWIbiK+u++OB3Er/wJr7ejTwP6XUBqXUdPd9iVrr4+7bJ4DEZsaY5L5d/f7mupOqf/z+9D5C67xvdR2jOe7D+PTs0VUptUkptUIpNaZS7E2NxRv/1lr6d1vxHPfjee7tm2oMkKm13lvpPn96H81Mztm1k3O2987ZYK7ztpyzCbxztpkSaZ9QSkUAnwJPaK3PADOA7sBg4DjGVwy+NFprfSFwDfCIUuqSyg+6P4lpn0RWiVIqGLgR+Nh9l7+9j1W0xvt2LsdQSj0POIH57ruOA5201kOAp4APlFJRrRFLLfz6d1vNRKomCv70PopmkHO2d5jtnA3+fd6Wc7bX+N0520yJ9FGMwSQeye77WoxSyoZxQp6vtf4MQGudqbUu11q7gFkYX3HUF19d95/G+NogqNr9TaK1Puq+zsIYyDAcyPR8HeG+zmpmjEep+jXUubzn1wAbtdaZ7nj96n10a433ra5jNJpSaipwPTDZfRLA/dXbafftDRj9a72aGcs5/Vtrpd9txXPcj0e7t2809/NuwRjE4ondb97HACDn7FrIOdur52wwwXlbztlV9xVo52wzJdLrgJ5Kqa7uT8p3Al+21MHcfTizgV1a65cr3V+5X+ZmYLv79pfAnUqpEKVUV6AnRqN7rXG7/zEtB25zP38K8EUTYwxXSkV6bmMMatjujmVKLfv9ErhHGUYCee6vNxYDVymlYt1f6VwFLHY/dkYpNdL9ftzT1BgrqfIp0p/ex0pa432r6xiNopQaB/wEuFFrXVTp/jZKKav7djeM9+1AM2Op6/U2NsbW+N1Wjv02YJnnP6gmuALYrbWu+PrPn97HACDn7Joxyjnbu+dsz/H99rwt5+zz4Jytmzma1xcXjBGV6RifOJ5v4WONxij3b6XSlDDAPIzpVLa63/T2lZ7zvDu2PVQaKV1X3BgjXtdiNPB/DIQ0McZuGKNlt2BM9/S8+/54YCnG1C5LgDj3/Qp4wx3HNiCl0r7uc8exD7i30v0pGP+o9gOv08SplNz7CMf45Bld6T6fvo8Y/0EcBxwYfVD3t8b7VtcxmhDjPoweLs/fpGcU9K3uv4HNwEbghubGUt/rbWSMLf67Bezun/e5H+/WlBjd978HPFhtW5+8j4F6qev32kLHknN2PX+nTYzT787Z7uf59Xm7jvjknK0D+5wtKxsKIYQQQgjRDGZq7RBCCCGEEMJvSCIthBBCCCFEM0giLYQQQgghRDNIIi2EEEIIIUQzSCIthBBCCCFEM0giLUxPKfWEUirM13EIIYRomJyzRSCR6e+E6SmlDmLM93jK17EIIYSon5yzRSCRirQwFffKYP9RSm1RSm1XSv0K6AAsV0otd29zlVLqe6XURqXUx0qpCPf9B5VSf1JKbVNKrVVK9fDlaxFCiEAn52wR6CSRFmYzDjimtR6kte4PvAIcAy7VWl+qlEoAfgFcobW+EFgPPFXp+Xla6wEYqxy90qqRCyHE+UfO2SKgSSItzGYbcKVS6iWl1BitdV61x0cCfYHvlFKbgSlA50qPL6h0PaqlgxVCiPOcnLNFQAvydQBCNIXWOl0pdSFwLfA7pdTSapso4Gut9cS6dlHHbSGEEF4m52wR6KQiLUxFKdUBKNJa/wP4M3AhkA9EujdZDVzs6aVz9+f1qrSLCZWuv2+dqIUQ4vwk52wR6KQiLcxmAPBnpZQLcAAPYXzd91+l1DF3z91UYIFSKsT9nF8A6e7bsUqprUApUFcFRAghhHfIOVsENJn+Tpw3ZMolIYQwDzlnCzOQ1g4hhBBCCCGaQSrSQgghhBBCNINUpIUQQgghhGgGSaSFEEIIIYRoBkmkhRBCCCGEaAZJpIUQQgghhGgGSaSFEEIIIYRoBkmkhRBCCCGEaIb/B2gsx0ZerfceAAAAAElFTkSuQmCC\n"
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#画线要注意的是损失是不一定在零到1之间的\n",
    "def plot_learning_curves(record_dict, sample_step=500):\n",
    "    # build DataFrame\n",
    "    train_df = pd.DataFrame(record_dict[\"train\"]).set_index(\"step\").iloc[::sample_step]\n",
    "    val_df = pd.DataFrame(record_dict[\"val\"]).set_index(\"step\")\n",
    "\n",
    "    # plot\n",
    "    fig_num = len(train_df.columns)\n",
    "    fig, axs = plt.subplots(1, fig_num, figsize=(6 * fig_num, 5))\n",
    "    for idx, item in enumerate(train_df.columns):    \n",
    "        axs[idx].plot(train_df.index, train_df[item], label=f\"train_{item}\")\n",
    "        axs[idx].plot(val_df.index, val_df[item], label=f\"val_{item}\")\n",
    "        axs[idx].grid()\n",
    "        axs[idx].legend()\n",
    "        axs[idx].set_xlabel(\"step\")\n",
    "    \n",
    "    plt.show()\n",
    "\n",
    "plot_learning_curves(record, sample_step=10000)  #横坐标是 steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-18T08:35:34.614992100Z",
     "start_time": "2024-07-18T08:35:30.962018300Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss:     0.3088\n",
      "accuracy: 0.8919\n"
     ]
    }
   ],
   "source": [
    "# dataload for evaluating\n",
    "model=model.to(device)\n",
    "# load checkpoints\n",
    "model.load_state_dict(torch.load(\"checkpoints/bn/best.ckpt\", map_location=\"cpu\"))\n",
    "\n",
    "model.eval()\n",
    "loss, acc = evaluating(model, val_loader, loss_fct)\n",
    "print(f\"loss:     {loss:.4f}\\naccuracy: {acc:.4f}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
